{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Alzheimers_Classification_PRML.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "Za608IEccbSn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ea1ZE70KchjV",
        "colab_type": "text"
      },
      "source": [
        "**Install dependencies**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xdIGchrwcgBP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip install scikit-plot"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ytw4UQfickda",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.metrics import accuracy_score\n",
        "from sklearn.svm import SVR\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "from sklearn import metrics\n",
        "from sklearn.metrics import explained_variance_score\n",
        "from sklearn.feature_selection import RFE\n",
        "import statistics \n",
        "from sklearn.ensemble import AdaBoostClassifier\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "from sklearn.linear_model import SGDClassifier\n",
        "import matplotlib.pyplot as plt\n",
        "from sklearn.discriminant_analysis import LinearDiscriminantAnalysis as LDA\n",
        "from mlxtend.feature_selection import SequentialFeatureSelector as SFS\n",
        "from sklearn.metrics import confusion_matrix\n",
        "from os.path import dirname, join as pjoin\n",
        "import scipy.io as sio\n",
        "from sklearn.neighbors import KNeighborsClassifier\n",
        "from sklearn import tree\n",
        "from sklearn.metrics import f1_score\n",
        "from sklearn import svm\n",
        "import numpy as np\n",
        "from sklearn.metrics import mean_absolute_error\n",
        "from sklearn.svm import SVC\n",
        "from collections import Counter\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.ensemble import StackingClassifier\n",
        "from matplotlib import pyplot\n",
        "from sklearn.tree import DecisionTreeClassifier\n",
        "import math"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mLNN1CBFcpCm",
        "colab_type": "text"
      },
      "source": [
        "**Data is stored inside the /PRML/ALZHEIMERS/ALZHEIMERS/ALZHEIMERS/ subdirectory of the mounted Google drive**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yNcbGFNOcmiB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "data_base_dir = '/content/drive/My Drive/PRML/ALZHEIMERS/ALZHEIMERS/ALZHEIMERS/'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "txkYZTEvc0pq",
        "colab_type": "text"
      },
      "source": [
        "*   **Implementing the LDA, Logistic regression, SVM, KNN and Decision tree classifiers**\n",
        "*   **Evaluating the train and test accuracy, sensitivity, specificity, mean absolute error, F1 score and confusion matrices of each of these classifiers on the AD-CTL, AD-MCI and MCI-CTL data**\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oE39RKyDcye6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Create lists to store the final accuracies, specificity, sensitivity, mean absolute error, F1 scores and confusion matrices for each classifier\n",
        "final_LDA_train_acc = []\n",
        "final_LDA_train_sens = []\n",
        "final_LDA_train_spec = []\n",
        "final_LDA_train_err = []\n",
        "final_LDA_train_f1 = [] \n",
        "final_LDA_test_acc = []\n",
        "final_LDA_test_sens = []\n",
        "final_LDA_test_spec = []\n",
        "final_LDA_test_err = []\n",
        "final_LDA_test_f1 = []\n",
        "final_SGD_train_acc = []\n",
        "final_SGD_train_sens = []\n",
        "final_SGD_train_spec = []\n",
        "final_SGD_train_err = []\n",
        "final_SGD_train_f1 = []\n",
        "final_SGD_test_acc = []\n",
        "final_SGD_test_sens = []\n",
        "final_SGD_test_spec = []\n",
        "final_SGD_test_err = []\n",
        "final_SGD_test_f1 = []\n",
        "final_SVM_train_acc = []\n",
        "final_SVM_train_sens = []\n",
        "final_SVM_train_spec = []\n",
        "final_SVM_train_err = []\n",
        "final_SVM_train_f1 = []\n",
        "final_SVM_test_acc = []\n",
        "final_SVM_test_sens = []\n",
        "final_SVM_test_spec = []\n",
        "final_SVM_test_err = []\n",
        "final_SVM_test_f1 = []\n",
        "final_DT_train_acc = []\n",
        "final_DT_train_sens = []\n",
        "final_DT_train_spec = []\n",
        "final_DT_train_err = []\n",
        "final_DT_train_f1 = []\n",
        "final_DT_test_acc = []\n",
        "final_DT_test_sens = []\n",
        "final_DT_test_spec = []\n",
        "final_DT_test_err = []\n",
        "final_DT_test_f1 = []\n",
        "final_KNN_train_acc = []\n",
        "final_KNN_train_sens = []\n",
        "final_KNN_train_spec = []\n",
        "final_KNN_train_err = []\n",
        "final_KNN_train_f1 = []\n",
        "final_KNN_test_acc = []\n",
        "final_KNN_test_sens = []\n",
        "final_KNN_test_spec = []\n",
        "final_KNN_test_err = []\n",
        "final_KNN_test_f1 = []\n",
        "final_LDA_test_std = []\n",
        "final_SGD_test_std = []\n",
        "final_SVM_test_std = []\n",
        "final_DT_test_std = []\n",
        "final_KNN_test_std = []\n",
        "TN_lda = []\n",
        "TP_lda = []\n",
        "FN_lda = []\n",
        "FP_lda = []\n",
        "TN_sgd = []\n",
        "TP_sgd = []\n",
        "FN_sgd = []\n",
        "FP_sgd = []\n",
        "TN_svm = []\n",
        "TP_svm = []\n",
        "FN_svm = []\n",
        "FP_svm = []\n",
        "TN_knn = []\n",
        "TP_knn = []\n",
        "FN_knn = []\n",
        "FP_knn = []\n",
        "TN_dt = []\n",
        "TP_dt = []\n",
        "FN_dt = []\n",
        "FP_dt = []\n",
        "dir_files = ['/AD_CTL', '/AD_MCI', '/MCI_CTL']\n",
        "mat_files = ['AD.CTL.0.aging.lookupFeatures.1000.perm1.split', 'AD.MCI.0.aging.lookupFeatures.1000.perm1.split', \n",
        "             'MCI.CTL.0.aging.lookupFeatures.1000.perm1.split']\n",
        "# Iterate through each of the 40 cross validation splits             \n",
        "for i in range(2,42):\n",
        "    data_dir = data_base_dir + '/SPLIT_' + str(i) \n",
        "    # Create lists to store the accuracies, specificity, sensitivity, mean absolute error, F1 scores and confusion matrices for each data split\n",
        "    temp_LDA_train_acc = []\n",
        "    temp_LDA_train_sens = []\n",
        "    temp_LDA_train_spec = []\n",
        "    temp_LDA_train_err = []\n",
        "    temp_LDA_train_f1 = [] \n",
        "    temp_LDA_test_acc = []\n",
        "    temp_LDA_test_sens = []\n",
        "    temp_LDA_test_spec = []\n",
        "    temp_LDA_test_err = []\n",
        "    temp_LDA_test_f1 = []\n",
        "    temp_SGD_train_acc = []\n",
        "    temp_SGD_train_sens = []\n",
        "    temp_SGD_train_spec = []\n",
        "    temp_SGD_train_err = []\n",
        "    temp_SGD_train_f1 = []\n",
        "    temp_SGD_test_acc = []\n",
        "    temp_SGD_test_sens = []\n",
        "    temp_SGD_test_spec = []\n",
        "    temp_SGD_test_err = []\n",
        "    temp_SGD_test_f1 = []\n",
        "    temp_SVM_train_acc = []\n",
        "    temp_SVM_train_sens = []\n",
        "    temp_SVM_train_spec = []\n",
        "    temp_SVM_train_err = []\n",
        "    temp_SVM_train_f1 = []\n",
        "    temp_SVM_test_acc = []\n",
        "    temp_SVM_test_sens = []\n",
        "    temp_SVM_test_spec = []\n",
        "    temp_SVM_test_err = []\n",
        "    temp_SVM_test_f1 = []\n",
        "    temp_DT_train_acc = []\n",
        "    temp_DT_train_sens = []\n",
        "    temp_DT_train_spec = []\n",
        "    temp_DT_train_err = []\n",
        "    temp_DT_train_f1 = []\n",
        "    temp_DT_test_acc = []\n",
        "    temp_DT_test_sens = []\n",
        "    temp_DT_test_spec = []\n",
        "    temp_DT_test_err = []\n",
        "    temp_DT_test_f1 = []\n",
        "    temp_KNN_train_acc = []\n",
        "    temp_KNN_train_sens = []\n",
        "    temp_KNN_train_spec = []\n",
        "    temp_KNN_train_err = []\n",
        "    temp_KNN_train_f1 = []\n",
        "    temp_KNN_test_acc = []\n",
        "    temp_KNN_test_sens = []\n",
        "    temp_KNN_test_spec = []\n",
        "    temp_KNN_test_err = []\n",
        "    temp_KNN_test_f1 = []\n",
        "    temp_LDA_test_std = []\n",
        "    temp_SGD_test_std = []\n",
        "    temp_SVM_test_std = []\n",
        "    temp_DT_test_std = []\n",
        "    temp_KNN_test_std = []\n",
        "    # Iterate through the AD-CTL, AD-MCI and MCI-CTL data\n",
        "    for j in range(3):\n",
        "      data_dir_1 = data_dir + dir_files[j]\n",
        "      file_1 = mat_files[j] + str(i) + '.mat'\n",
        "      mat_fname_1 = pjoin(data_dir_1, file_1)\n",
        "      mat_1 = sio.loadmat(mat_fname_1)\n",
        "      mat_1_train = mat_1['trainingSpread']\n",
        "      mat_1_test = mat_1['testSpread']\n",
        "\n",
        "      x_train_1 = mat_1_train[:, :-1]\n",
        "      y_train_1 = mat_1_train[:, -1]\n",
        "      x_test_1 = mat_1_test[:, :-1]\n",
        "      y_test_1 = mat_1_test[:, -1]\n",
        "      X_train = x_train_1\n",
        "      X_test = x_test_1\n",
        "\n",
        "      # Implement recursive feature elimination using the SVM linear kernel to get the topmost discriminative features\n",
        "      svm = SVR(kernel='linear', gamma='scale')\n",
        "      selector = RFE(svm, 900)\n",
        "      selector = selector.fit(x_train_1, y_train_1)\n",
        "      ft_ind = selector.get_support(1)\n",
        "      final_X_train = x_train_1[:,ft_ind]\n",
        "      final_X_test = x_test_1[:,ft_ind]\n",
        "      x_train_1 = final_X_train\n",
        "      x_test_1 = final_X_test\n",
        "\n",
        "      # Combine output of RFE - SVM with each feature's covariance matrix\n",
        "      cov_train_f = np.zeros((len(x_train_1),1))\n",
        "      for j in range(900):\n",
        "          cov_train = np.zeros((len(x_train_1),1))\n",
        "          cov_val = np.cov(x_train_1[:,j])\n",
        "          temp = []\n",
        "          for i in range(len(x_train_1)):\n",
        "              cov_train[i] = x_train_1[:,j][i] + cov_val\n",
        "          cov_train_f = np.column_stack((cov_train_f,cov_train))\n",
        "          final_cov_train = np.delete(cov_train_f, 0, axis=1)\n",
        "\n",
        "      cov_test_f = np.zeros((len(x_test_1),1))\n",
        "      for j in range(900):\n",
        "          cov_test = np.zeros((len(x_test_1),1))\n",
        "          cov_val = np.cov(x_test_1[:,j])\n",
        "          temp = []\n",
        "          for i in range(len(x_test_1)):\n",
        "              cov_test[i] = x_test_1[:,j][i] + cov_val\n",
        "          cov_test_f = np.column_stack((cov_test_f,cov_test))\n",
        "          final_cov_test = np.delete(cov_test_f, 0, axis=1)    \n",
        "\n",
        "      # Implement sequential forward selection on these features combined with each covariance matrix using the SVM linear kernel to get \n",
        "      # the 800 highly discriminative features\n",
        "      sfs = SFS(SVC(gamma='auto'), k_features=800, forward=True, floating=False, scoring = 'accuracy', cv = 0)\n",
        "      sfs.fit(final_cov_train, y_train_1)\n",
        "      sfs_ind = sfs.k_feature_idx_\n",
        "      final_X_train = x_train_1[:,sfs_ind] \n",
        "\n",
        "      sfs.fit(final_cov_test, y_test_1)\n",
        "      sfs_ind = sfs.k_feature_idx_\n",
        "      final_X_test = x_test_1[:,sfs_ind] \n",
        "\n",
        "      # Implement LDA classifier\n",
        "      sc = StandardScaler()\n",
        "      X_train = sc.fit_transform(final_X_train)\n",
        "      X_test = sc.transform(final_X_test)\n",
        "\n",
        "      lda = LDA()\n",
        "      lda.fit(X_train, y_train_1)\n",
        "    \n",
        "      # Compute training accuracy \n",
        "      ypred_train = lda.predict(X_train)\n",
        "      acc_train = accuracy_score(y_true=y_train_1, y_pred=ypred_train)\n",
        "      temp_LDA_train_acc.append(acc_train)\n",
        "      print('Training Accuracy: {:.4f}'.format(acc_train))\n",
        "      # Compute training specificity\n",
        "      tn, fp, fn, tp = confusion_matrix(y_train_1, ypred_train).ravel()\n",
        "      print(tn,fp,fn,tp)\n",
        "      specificity_train = tn / (tn+fp)\n",
        "      temp_LDA_train_spec.append(specificity_train)\n",
        "      print('Training Specificity: {:.4f}'.format(specificity_train))\n",
        "      # Compute training sensitivity\n",
        "      sensitivity_train = tp/ (tp+fn)\n",
        "      temp_LDA_train_sens.append(sensitivity_train)\n",
        "      print('Training Sensitivity: {:.4f}'.format(sensitivity_train))\n",
        "      # Compute training mean absolute error\n",
        "      err_train = mean_absolute_error(y_train_1, ypred_train)\n",
        "      temp_LDA_train_err.append(err_train)\n",
        "      print('Mean abs error: {:.4f}'.format(err_train))\n",
        "      # Compute training F1 score\n",
        "      p = tp/(tp+fp)\n",
        "      r = tp/(tp+fn)\n",
        "      f1_train = (2*p*r)/(r+p)\n",
        "      temp_LDA_train_f1.append(f1_train)\n",
        "      print('F1 score: {:.4f}'.format(f1_train))\n",
        "\n",
        "      # Compute test accuracy \n",
        "      ypred_test = lda.predict(X_test)\n",
        "      acc_test = accuracy_score(y_true=y_test_1, y_pred=ypred_test)\n",
        "      print('Test Accuracy: {:.4f}'.format(acc_test))\n",
        "      # Compute test specificity\n",
        "      temp_LDA_test_acc.append(acc_test)\n",
        "      tn, fp, fn, tp = confusion_matrix(y_test_1, ypred_test).ravel()\n",
        "      TN_lda.append(tn)\n",
        "      TP_lda.append(tp)\n",
        "      FP_lda.append(fp)\n",
        "      FN_lda.append(fn)\n",
        "      print(tn,fp,fn,tp)\n",
        "      specificity_test = tn / (tn+fp)\n",
        "      temp_LDA_test_spec.append(specificity_test)\n",
        "      print('Test Specificity: {:.4f}'.format(specificity_test))\n",
        "      # Compute test sensitivity\n",
        "      sensitivity_test = tp/ (tp+fn)\n",
        "      temp_LDA_test_sens.append(sensitivity_test)\n",
        "      print('Test Sensitivity: {:.4f}'.format(sensitivity_test))\n",
        "      # Compute test mean absolute error\n",
        "      err_test = mean_absolute_error(y_test_1, ypred_test)\n",
        "      temp_LDA_test_err.append(err_test)\n",
        "      print('Mean abs error: {:.4f}'.format(err_test))\n",
        "      # Compute test F1 score\n",
        "      p = tp/(tp+fp)\n",
        "      r = tp/(tp+fn)\n",
        "      f1_test = (2*p*r)/(r+p)\n",
        "      temp_LDA_test_f1.append(f1_test)\n",
        "      print('F1 score: {:.4f}'.format(f1_test))\n",
        "\n",
        "      # Compute Logistic regression classifier\n",
        "      classifier = SGDClassifier(loss='log', penalty=None, max_iter=80000, tol=None, learning_rate='constant', eta0=1e-4)\n",
        "      classifier.fit(X_train, y_train_1)\n",
        "      print('Train accuracy = {}%'.format(100*classifier.score(X_train, y_train_1)))\n",
        "      print('Test accuracy = {}%'.format(100*classifier.score(X_test, y_test_1)))\n",
        "      # Compute training accuracy\n",
        "      ypred_train = classifier.predict(X_train)\n",
        "      acc_train = accuracy_score(y_true=y_train_1, y_pred=ypred_train)\n",
        "      print('Training Accuracy: {:.4f}'.format(acc_train))\n",
        "      temp_SGD_train_acc.append(acc_train)\n",
        "      # Compute training specificity\n",
        "      tn, fp, fn, tp = confusion_matrix(y_train_1, ypred_train).ravel()\n",
        "      print(tn,fp,fn,tp)\n",
        "      specificity_train = tn / (tn+fp)\n",
        "      temp_SGD_train_spec.append(specificity_train)\n",
        "      print('Training Specificity: {:.4f}'.format(specificity_train))\n",
        "      # Compute training sensitivity\n",
        "      sensitivity_train = tp/ (tp+fn)\n",
        "      temp_SGD_train_sens.append(sensitivity_train)\n",
        "      print('Training Sensitivity: {:.4f}'.format(sensitivity_train))\n",
        "      # Compute training mean absolute error\n",
        "      err_train = mean_absolute_error(y_train_1, ypred_train)\n",
        "      temp_SGD_train_err.append(err_train)\n",
        "      print('Mean abs error: {:.4f}'.format(err_train))\n",
        "      # Compute training F1 score\n",
        "      p = tp/(tp+fp)\n",
        "      r = tp/(tp+fn)\n",
        "      f1_train = (2*p*r)/(r+p)\n",
        "      temp_SGD_train_f1.append(f1_train)\n",
        "      print('F1 score: {:.4f}'.format(f1_train))\n",
        "\n",
        "      # Compute test accuracy\n",
        "      ypred_test = classifier.predict(X_test)\n",
        "      acc_test = accuracy_score(y_true=y_test_1, y_pred=ypred_test)\n",
        "      print('Test Accuracy: {:.4f}'.format(acc_test))\n",
        "      temp_SGD_test_acc.append(acc_test)\n",
        "      # Compute test specificity\n",
        "      tn, fp, fn, tp = confusion_matrix(y_test_1, ypred_test).ravel()\n",
        "      TN_sgd.append(tn)\n",
        "      TP_sgd.append(tp)\n",
        "      FP_sgd.append(fp)\n",
        "      FN_sgd.append(fn)\n",
        "      print(tn,fp,fn,tp)\n",
        "      specificity_test = tn / (tn+fp)\n",
        "      temp_SGD_test_spec.append(specificity_test)\n",
        "      print('Test Specificity: {:.4f}'.format(specificity_test))\n",
        "      # Compute test sensitivity\n",
        "      sensitivity_test = tp/ (tp+fn)\n",
        "      temp_SGD_test_sens.append(sensitivity_test)\n",
        "      print('Test Sensitivity: {:.4f}'.format(sensitivity_test))\n",
        "      # Compute test mean absolute error\n",
        "      err_test = mean_absolute_error(y_test_1, ypred_test)\n",
        "      temp_SGD_test_err.append(err_test)\n",
        "      print('Mean abs error: {:.4f}'.format(err_test))\n",
        "      # Compute test F1 score\n",
        "      p = tp/(tp+fp)\n",
        "      r = tp/(tp+fn)\n",
        "      f1_test = (2*p*r)/(r+p)\n",
        "      temp_SGD_test_f1.append(f1_test)\n",
        "      print('F1 score: {:.4f}'.format(f1_test))\n",
        "     \n",
        "      # Implement SVM classifier\n",
        "      clf = SVC()\n",
        "      clf.fit(X_train, y_train_1)\n",
        "      print('Train accuracy = {}%'.format(100*clf.score(X_train, y_train_1)))\n",
        "      print('Test accuracy = {}%'.format(100*clf.score(X_test, y_test_1)))\n",
        "      # Compute training accuracy\n",
        "      ypred_train = clf.predict(X_train)\n",
        "      acc_train = accuracy_score(y_true=y_train_1, y_pred=ypred_train)\n",
        "      temp_SVM_train_acc.append(acc_train)\n",
        "      print('Training Accuracy: {:.4f}'.format(acc_train))\n",
        "      # Compute training specificity\n",
        "      tn, fp, fn, tp = confusion_matrix(y_train_1, ypred_train).ravel()\n",
        "      print(tn,fp,fn,tp)\n",
        "      specificity_train = tn / (tn+fp)\n",
        "      temp_SVM_train_spec.append(specificity_train)\n",
        "      print('Training Specificity: {:.4f}'.format(specificity_train))\n",
        "      # Compute training sensitivity\n",
        "      sensitivity_train = tp/ (tp+fn)\n",
        "      temp_SVM_train_sens.append(sensitivity_train)\n",
        "      print('Training Sensitivity: {:.4f}'.format(sensitivity_train))\n",
        "      # Compute training mean absolute error\n",
        "      err_train = mean_absolute_error(y_train_1, ypred_train)\n",
        "      temp_SVM_train_err.append(err_train)\n",
        "      print('Mean abs error: {:.4f}'.format(err_train))\n",
        "      # Compute training F1 score\n",
        "      p = tp/(tp+fp)\n",
        "      r = tp/(tp+fn)\n",
        "      f1_train = (2*p*r)/(r+p)\n",
        "      temp_SVM_train_f1.append(f1_train)\n",
        "      print('F1 score: {:.4f}'.format(f1_train))\n",
        "\n",
        "      # Compute test accuracy\n",
        "      ypred_test = clf.predict(X_test)\n",
        "      acc_test = accuracy_score(y_true=y_test_1, y_pred=ypred_test)\n",
        "      temp_SVM_test_acc.append(acc_test)\n",
        "      print('Test Accuracy: {:.4f}'.format(acc_test))\n",
        "      # Compute test specificity\n",
        "      tn, fp, fn, tp = confusion_matrix(y_test_1, ypred_test).ravel()\n",
        "      TN_svm.append(tn)\n",
        "      TP_svm.append(tp)\n",
        "      FP_svm.append(fp)\n",
        "      FN_svm.append(fn)\n",
        "      print(tn,fp,fn,tp)\n",
        "      specificity_test = tn / (tn+fp)\n",
        "      temp_SVM_test_spec.append(specificity_test)\n",
        "      print('Test Specificity: {:.4f}'.format(specificity_test))\n",
        "      # Compute test sensitivity\n",
        "      sensitivity_test = tp/ (tp+fn)\n",
        "      temp_SVM_test_sens.append(sensitivity_test)\n",
        "      print('Test Sensitivity: {:.4f}'.format(sensitivity_test))\n",
        "      # Compute test mean absolute error\n",
        "      err_test = mean_absolute_error(y_test_1, ypred_test)\n",
        "      temp_SVM_test_err.append(err_test)\n",
        "      print('Mean abs error: {:.4f}'.format(err_test))\n",
        "      # Compute test F1 score\n",
        "      p = tp/(tp+fp)\n",
        "      r = tp/(tp+fn)\n",
        "      f1_test = (2*p*r)/(r+p)\n",
        "      temp_SVM_test_f1.append(f1_test)\n",
        "      print('F1 score: {:.4f}'.format(f1_test))\n",
        "\n",
        "      # Implement Decision tree classifier\n",
        "      dtc = tree.DecisionTreeClassifier(max_depth=5)\n",
        "      dtc.fit(X_train, y_train_1)\n",
        "      train_acc = dtc.score(X_train, y_train_1)\n",
        "      test_acc = dtc.score(X_test, y_test_1)\n",
        "      print(\"Training accuracy: \", train_acc * 100, \"%\")\n",
        "      print(\"Test accuracy: \", test_acc * 100, \"%\")\n",
        "      # Compute training accuracy\n",
        "      ypred_train = dtc.predict(X_train)\n",
        "      acc_train = accuracy_score(y_true=y_train_1, y_pred=ypred_train)\n",
        "      temp_DT_train_acc.append(acc_train)\n",
        "      print('Training Accuracy: {:.4f}'.format(acc_train))\n",
        "      # Compute training specificity\n",
        "      tn, fp, fn, tp = confusion_matrix(y_train_1, ypred_train).ravel()\n",
        "      print(tn,fp,fn,tp)\n",
        "      specificity_train = tn / (tn+fp)\n",
        "      temp_DT_train_spec.append(specificity_train)\n",
        "      print('Training Specificity: {:.4f}'.format(specificity_train))\n",
        "      # Compute training sensitivity\n",
        "      sensitivity_train = tp/ (tp+fn)\n",
        "      temp_DT_train_sens.append(sensitivity_train)\n",
        "      print('Training Sensitivity: {:.4f}'.format(sensitivity_train))\n",
        "      # Compute training mean absolute error\n",
        "      err_train = mean_absolute_error(y_train_1, ypred_train)\n",
        "      temp_DT_train_err.append(err_train)\n",
        "      print('Mean abs error: {:.4f}'.format(err_train))\n",
        "      # Compute training F1 score\n",
        "      p = tp/(tp+fp)\n",
        "      r = tp/(tp+fn)\n",
        "      f1_train = (2*p*r)/(r+p)\n",
        "      temp_DT_train_f1.append(f1_train)\n",
        "      print('F1 score: {:.4f}'.format(f1_train))\n",
        "\n",
        "      # Compute test accuracy\n",
        "      ypred_test = dtc.predict(X_test)\n",
        "      acc_test = accuracy_score(y_true=y_test_1, y_pred=ypred_test)\n",
        "      temp_DT_test_acc.append(acc_test)\n",
        "      print('Test Accuracy: {:.4f}'.format(acc_test))\n",
        "      # Compute test specificity\n",
        "      tn, fp, fn, tp = confusion_matrix(y_test_1, ypred_test).ravel()\n",
        "      TN_dt.append(tn)\n",
        "      TP_dt.append(tp)\n",
        "      FP_dt.append(fp)\n",
        "      FN_dt.append(fn)\n",
        "      print(tn,fp,fn,tp)\n",
        "      specificity_test = tn / (tn+fp)\n",
        "      temp_DT_test_spec.append(specificity_test)\n",
        "      print('Test Specificity: {:.4f}'.format(specificity_test))\n",
        "      # Compute test sensitivity\n",
        "      sensitivity_test = tp/ (tp+fn)\n",
        "      temp_DT_test_sens.append(sensitivity_test)\n",
        "      print('Test Sensitivity: {:.4f}'.format(sensitivity_test))\n",
        "      # Compute test mean absolute error\n",
        "      err_test = mean_absolute_error(y_test_1, ypred_test)\n",
        "      temp_DT_test_err.append(err_test)\n",
        "      print('Mean abs error: {:.4f}'.format(err_test))\n",
        "      # Compute test F1 score\n",
        "      p = tp/(tp+fp)\n",
        "      r = tp/(tp+fn)\n",
        "      f1_test = (2*p*r)/(r+p)\n",
        "      temp_DT_test_f1.append(f1_test)\n",
        "      print('F1 score: {:.4f}'.format(f1_test))\n",
        "\n",
        "      # Implement KNN classifier\n",
        "      neigh = KNeighborsClassifier(n_neighbors=28)\n",
        "      neigh.fit(X_train, y_train_1)\n",
        "      train_acc = dtc.score(X_train, y_train_1)\n",
        "      test_acc = dtc.score(X_test, y_test_1)\n",
        "      print(\"Training accuracy: \", train_acc * 100, \"%\")\n",
        "      print(\"Test accuracy: \", test_acc * 100, \"%\")\n",
        "      # Compute training accuracy\n",
        "      ypred_train = dtc.predict(X_train)\n",
        "      acc_train = accuracy_score(y_true=y_train_1, y_pred=ypred_train)\n",
        "      temp_KNN_train_acc.append(acc_train)\n",
        "      print('Training Accuracy: {:.4f}'.format(acc_train))\n",
        "      # Compute training specificity\n",
        "      tn, fp, fn, tp = confusion_matrix(y_train_1, ypred_train).ravel()\n",
        "      TN_knn.append(tn)\n",
        "      TP_knn.append(tp)\n",
        "      FP_knn.append(fp)\n",
        "      FN_knn.append(fn)\n",
        "      print(tn,fp,fn,tp)\n",
        "      specificity_train = tn / (tn+fp)\n",
        "      temp_KNN_train_spec.append(specificity_train)\n",
        "      print('Training Specificity: {:.4f}'.format(specificity_train))\n",
        "      # Compute training sensitivity\n",
        "      sensitivity_train = tp/ (tp+fn)\n",
        "      temp_KNN_train_sens.append(sensitivity_train)\n",
        "      print('Training Sensitivity: {:.4f}'.format(sensitivity_train))\n",
        "      # Compute training mean absolute error\n",
        "      err_train = mean_absolute_error(y_train_1, ypred_train)\n",
        "      temp_KNN_train_err.append(err_train)\n",
        "      print('Mean abs error: {:.4f}'.format(err_train))\n",
        "      # Compute training F1 score\n",
        "      p = tp/(tp+fp)\n",
        "      r = tp/(tp+fn)\n",
        "      f1_train = (2*p*r)/(r+p)\n",
        "      temp_KNN_train_f1.append(f1_train)\n",
        "      print('F1 score: {:.4f}'.format(f1_train))\n",
        "\n",
        "      # Compute test accuracy\n",
        "      ypred_test = dtc.predict(X_test)\n",
        "      acc_test = accuracy_score(y_true=y_test_1, y_pred=ypred_test)\n",
        "      temp_KNN_test_acc.append(acc_test)\n",
        "      print('Test Accuracy: {:.4f}'.format(acc_test))\n",
        "      # Compute test specificity\n",
        "      tn, fp, fn, tp = confusion_matrix(y_test_1, ypred_test).ravel()\n",
        "      print(tn,fp,fn,tp)\n",
        "      specificity_test = tn / (tn+fp)\n",
        "      temp_KNN_test_spec.append(specificity_test)\n",
        "      print('Test Specificity: {:.4f}'.format(specificity_test))\n",
        "      # Compute test sensitivity\n",
        "      sensitivity_test = tp/ (tp+fn)\n",
        "      temp_KNN_test_sens.append(sensitivity_test)\n",
        "      print('Test Sensitivity: {:.4f}'.format(sensitivity_test))\n",
        "      # Compute test mean absolute error\n",
        "      err_test = mean_absolute_error(y_test_1, ypred_test)\n",
        "      temp_KNN_test_err.append(err_test)\n",
        "      print('Mean abs error: {:.4f}'.format(err_test))\n",
        "      # Compute test F1 score\n",
        "      p = tp/(tp+fp)\n",
        "      r = tp/(tp+fn)\n",
        "      f1_test = (2*p*r)/(r+p)\n",
        "      temp_KNN_test_f1.append(f1_test)\n",
        "      print('F1 score: {:.4f}'.format(f1_test))\n",
        "      \n",
        "    # Append all the values for each split into the respective final accuracy, sensitivity, specificity, mean absolute error and F1 scores lists \n",
        "    final_LDA_train_acc.append(temp_LDA_train_acc)\n",
        "    final_LDA_train_sens.append(temp_LDA_train_sens)\n",
        "    final_LDA_train_spec.append(temp_LDA_train_spec)\n",
        "    final_LDA_train_err.append(temp_LDA_train_err)\n",
        "    final_LDA_train_f1.append(temp_LDA_train_f1) \n",
        "    final_LDA_test_acc.append(temp_LDA_test_acc)\n",
        "    final_LDA_test_sens.append(temp_LDA_test_sens)\n",
        "    final_LDA_test_spec.append(temp_LDA_test_spec)\n",
        "    final_LDA_test_err.append(temp_LDA_test_err)\n",
        "    final_LDA_test_f1.append(temp_LDA_test_f1)\n",
        "    final_SGD_train_acc.append(temp_SGD_train_acc)\n",
        "    final_SGD_train_sens.append(temp_SGD_train_sens)\n",
        "    final_SGD_train_spec.append(temp_SGD_train_spec)\n",
        "    final_SGD_train_err.append(temp_SGD_train_err)\n",
        "    final_SGD_train_f1.append(temp_SGD_train_f1)\n",
        "    final_SGD_test_acc.append(temp_SGD_test_acc)\n",
        "    final_SGD_test_sens.append(temp_SGD_test_sens)\n",
        "    final_SGD_test_spec.append(temp_SGD_test_spec)\n",
        "    final_SGD_test_err.append(temp_SGD_test_err)\n",
        "    final_SGD_test_f1.append(temp_SGD_test_f1)\n",
        "    final_SVM_train_acc.append(temp_SVM_train_acc)\n",
        "    final_SVM_train_sens.append(temp_SVM_train_sens)\n",
        "    final_SVM_train_spec.append(temp_SVM_train_spec)\n",
        "    final_SVM_train_err.append(temp_SVM_train_err)\n",
        "    final_SVM_train_f1.append(temp_SVM_train_f1)\n",
        "    final_SVM_test_acc.append(temp_SVM_test_acc)\n",
        "    final_SVM_test_sens.append(temp_SVM_test_sens)\n",
        "    final_SVM_test_spec.append(temp_SVM_test_spec)\n",
        "    final_SVM_test_err.append(temp_SVM_test_err)\n",
        "    final_SVM_test_f1.append(temp_SVM_test_f1)\n",
        "    final_DT_train_acc.append(temp_DT_train_acc)\n",
        "    final_DT_train_sens.append(temp_DT_train_sens)\n",
        "    final_DT_train_spec.append(temp_DT_train_spec)\n",
        "    final_DT_train_err.append(temp_DT_train_err)\n",
        "    final_DT_train_f1.append(temp_DT_train_f1)\n",
        "    final_DT_test_acc.append(temp_DT_test_acc)\n",
        "    final_DT_test_sens.append(temp_DT_test_sens)\n",
        "    final_DT_test_spec.append(temp_DT_test_spec)\n",
        "    final_DT_test_err.append(temp_DT_test_err)\n",
        "    final_DT_test_f1.append(temp_DT_test_f1)\n",
        "    final_KNN_train_acc.append(temp_KNN_train_acc)\n",
        "    final_KNN_train_sens.append(temp_KNN_train_sens)\n",
        "    final_KNN_train_spec.append(temp_KNN_train_spec)\n",
        "    final_KNN_train_err.append(temp_KNN_train_err)\n",
        "    final_KNN_train_f1.append(temp_KNN_train_f1)\n",
        "    final_KNN_test_acc.append(temp_KNN_test_acc)\n",
        "    final_KNN_test_sens.append(temp_KNN_test_sens)\n",
        "    final_KNN_test_spec.append(temp_KNN_test_spec)\n",
        "    final_KNN_test_err.append(temp_KNN_test_err)\n",
        "    final_KNN_test_f1.append(temp_KNN_test_f1)\n",
        "    final_LDA_test_std.append(temp_LDA_test_std)\n",
        "    final_SGD_test_std.append(temp_LDA_test_std)\n",
        "    final_SVM_test_std.append(temp_LDA_test_std)\n",
        "    final_DT_test_std.append(temp_LDA_test_std)\n",
        "    final_KNN_test_std.append(temp_LDA_test_std)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KYJIWxe4c9Dj",
        "colab_type": "text"
      },
      "source": [
        "*   **Implementing the Stacking ensemble of classifiers**\n",
        "*   **Evaluating the train and test accuracy, sensitivity, specificity, mean absolute error, F1 score and confusion matrices of this classifier on the AD-CTL, AD-MCI and MCI-CTL data**\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yR4IUE2Sc6wk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Create lists to store the final accuracies, specificity, sensitivity, mean absolute error, F1 scores and confusion matrices for each classifier\n",
        "final_stack_test_acc = []\n",
        "final_stack_test_sens = []\n",
        "final_stack_test_spec = []\n",
        "final_stack_test_err = []\n",
        "final_stack_test_f1 = []\n",
        "final_stack_test_std = []\n",
        "TN_stack = []\n",
        "TP_stack = []\n",
        "FN_stack = []\n",
        "FP_stack = []\n",
        "\n",
        "dir_files = ['/AD_CTL', '/AD_MCI', '/MCI_CTL']\n",
        "mat_files = ['AD.CTL.0.aging.lookupFeatures.1000.perm1.split', 'AD.MCI.0.aging.lookupFeatures.1000.perm1.split', \n",
        "             'MCI.CTL.0.aging.lookupFeatures.1000.perm1.split']\n",
        "# Iterate through each of the 40 cross validation splits              \n",
        "for i in range(2,42):\n",
        "    data_dir = data_base_dir + '/SPLIT_' + str(i) \n",
        "    # Create lists to store the accuracies, specificity, sensitivity, mean absolute error, F1 scores and confusion matrices for each data split\n",
        "    temp_stack_test_acc = []\n",
        "    temp_stack_test_sens = []\n",
        "    temp_stack_test_spec = []\n",
        "    temp_stack_test_err = []\n",
        "    temp_stack_test_f1 = []\n",
        "    temp_stack_test_std = []\n",
        " \n",
        "    # Iterate through the AD-CTL, AD-MCI and MCI-CTL data\n",
        "    for j in range(3):\n",
        "      data_dir_1 = data_dir + dir_files[j]\n",
        "      file_1 = mat_files[j] + str(i) + '.mat'\n",
        "      mat_fname_1 = pjoin(data_dir_1, file_1)\n",
        "      mat_1 = sio.loadmat(mat_fname_1)\n",
        "      mat_1_train = mat_1['trainingSpread']\n",
        "      mat_1_test = mat_1['testSpread']\n",
        "\n",
        "      x_train_1 = mat_1_train[:, :-1]\n",
        "      y_train_1 = mat_1_train[:, -1]\n",
        "      x_test_1 = mat_1_test[:, :-1]\n",
        "      y_test_1 = mat_1_test[:, -1]\n",
        "      X_train = x_train_1\n",
        "      X_test = x_test_1\n",
        "\n",
        "      # Implement recursive feature elimination using the SVM linear kernel to get the topmost discriminative features\n",
        "      svm = SVR(kernel='linear', gamma='scale')\n",
        "      selector = RFE(svm, 900)\n",
        "      selector = selector.fit(x_train_1, y_train_1)\n",
        "      ft_ind = selector.get_support(1)\n",
        "      final_X_train = x_train_1[:,ft_ind]\n",
        "      final_X_test = x_test_1[:,ft_ind]\n",
        "      x_train_1 = final_X_train\n",
        "      x_test_1 = final_X_test\n",
        "\n",
        "      #Combine output of RFE - SVM with each feature's covariance matrix\n",
        "      cov_train_f = np.zeros((len(x_train_1),1))\n",
        "      for j in range(900):\n",
        "          cov_train = np.zeros((len(x_train_1),1))\n",
        "          cov_val = np.cov(x_train_1[:,j])\n",
        "          temp = []\n",
        "          for i in range(len(x_train_1)):\n",
        "              cov_train[i] = x_train_1[:,j][i] + cov_val\n",
        "          cov_train_f = np.column_stack((cov_train_f,cov_train))\n",
        "          final_cov_train = np.delete(cov_train_f, 0, axis=1)\n",
        "\n",
        "      cov_test_f = np.zeros((len(x_test_1),1))\n",
        "      for j in range(900):\n",
        "          cov_test = np.zeros((len(x_test_1),1))\n",
        "          cov_val = np.cov(x_test_1[:,j])\n",
        "          temp = []\n",
        "          for i in range(len(x_test_1)):\n",
        "              cov_test[i] = x_test_1[:,j][i] + cov_val\n",
        "          cov_test_f = np.column_stack((cov_test_f,cov_test))\n",
        "          final_cov_test = np.delete(cov_test_f, 0, axis=1)    \n",
        "\n",
        "      # Implement sequential forward selection on these features combined with each covariance matrix using the SVM linear kernel to get \n",
        "      # the 800 highly discriminative features\n",
        "      sfs = SFS(SVC(gamma='auto'), k_features=800, forward=True, floating=False, scoring = 'accuracy', cv = 0)\n",
        "      sfs.fit(final_cov_train, y_train_1)\n",
        "      sfs_ind = sfs.k_feature_idx_\n",
        "      final_X_train = x_train_1[:,sfs_ind] \n",
        "\n",
        "      sfs.fit(final_cov_test, y_test_1)\n",
        "      sfs_ind = sfs.k_feature_idx_\n",
        "      final_X_test = x_test_1[:,sfs_ind] \n",
        "\n",
        "      # Implement the stacking ensemble of classifiers \n",
        "      sc = StandardScaler()\n",
        "      X_train = sc.fit_transform(final_X_train)\n",
        "      X_test = sc.transform(final_X_test)\n",
        "\n",
        "      models = [('dt',DecisionTreeClassifier()), ('knn', KNeighborsClassifier(n_neighbors=24)), ('lda',LDA())]\n",
        "      stacking = StackingClassifier(estimators=models, final_estimator= LogisticRegression())\n",
        "      stacking.fit(X_train, y_train_1)\n",
        "    \n",
        "      # Compute test accuracy\n",
        "      ypred_test = stacking.predict(X_test)\n",
        "      acc_test = accuracy_score(y_true=y_test_1, y_pred=ypred_test)\n",
        "      print('Test Accuracy: {:.4f}'.format(acc_test))\n",
        "      temp_stack_test_acc.append(acc_test)\n",
        "      # Compute test specificity\n",
        "      tn, fp, fn, tp = confusion_matrix(y_test_1, ypred_test).ravel()\n",
        "      TN_stack.append(tn)\n",
        "      TP_stack.append(tp)\n",
        "      FP_stack.append(fp)\n",
        "      FN_stack.append(fn)\n",
        "      print(tn,fp,fn,tp)\n",
        "      specificity_test = tn / (tn+fp)\n",
        "      temp_stack_test_spec.append(specificity_test)\n",
        "      print('Test Specificity: {:.4f}'.format(specificity_test))\n",
        "      # Compute test sensitivity\n",
        "      sensitivity_test = tp/ (tp+fn)\n",
        "      temp_stack_test_sens.append(sensitivity_test)\n",
        "      print('Test Sensitivity: {:.4f}'.format(sensitivity_test))\n",
        "      # Compute test mean absolute error\n",
        "      err_test = mean_absolute_error(y_test_1, ypred_test)\n",
        "      temp_stack_test_err.append(err_test)\n",
        "      print('Mean abs error: {:.4f}'.format(err_test))\n",
        "      # Compute test F1 score\n",
        "      p = tp/(tp+fp)\n",
        "      r = tp/(tp+fn)\n",
        "      f1_test = (2*p*r)/(r+p)\n",
        "      temp_stack_test_f1.append(f1_test)\n",
        "      print('F1 score: {:.4f}'.format(f1_test))\n",
        "\n",
        "    # Append all the values for each split into the respective final accuracy, sensitivity, specificity, mean absolute error and F1 scores lists \n",
        "    final_stack_test_acc.append(temp_stack_test_acc)\n",
        "    final_stack_test_sens.append(temp_stack_test_sens)\n",
        "    final_stack_test_spec.append(temp_stack_test_spec)\n",
        "    final_stack_test_err.append(temp_stack_test_err)\n",
        "    final_stack_test_f1.append(temp_stack_test_f1)\n",
        "    final_stack_test_std.append(temp_stack_test_std)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jJqU-HTVdBSW",
        "colab_type": "text"
      },
      "source": [
        "*   **Implementing the Random forest classifier**\n",
        "*   **Evaluating the train and test accuracy, sensitivity, specificity, mean absolute error, F1 score and confusion matrices of this classifier on the AD-CTL, AD-MCI and MCI-CTL data**\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wgLhe4y0dDpy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Create lists to store the final accuracies, specificity, sensitivity, mean absolute error, F1 scores and confusion matrices for each classifier\n",
        "final_rf_test_acc = []\n",
        "final_rf_test_sens = []\n",
        "final_rf_test_spec = []\n",
        "final_rf_test_err = []\n",
        "final_rf_test_f1 = []\n",
        "final_rf_test_std = []\n",
        "TN_rf = []\n",
        "TP_rf = []\n",
        "FN_rf = []\n",
        "FP_rf = []\n",
        "dir_files = ['/AD_CTL', '/AD_MCI', '/MCI_CTL']\n",
        "mat_files = ['AD.CTL.0.aging.lookupFeatures.1000.perm1.split', 'AD.MCI.0.aging.lookupFeatures.1000.perm1.split', \n",
        "             'MCI.CTL.0.aging.lookupFeatures.1000.perm1.split']\n",
        "# Iterate through each of the 40 cross validation splits              \n",
        "for i in range(2,42):\n",
        "    data_dir = data_base_dir + '/SPLIT_' + str(i) \n",
        "    # Create lists to store the accuracies, specificity, sensitivity, mean absolute error, F1 scores and confusion matrices for each data split\n",
        "    temp_rf_test_acc = []\n",
        "    temp_rf_test_sens = []\n",
        "    temp_rf_test_spec = []\n",
        "    temp_rf_test_err = []\n",
        "    temp_rf_test_f1 = []\n",
        "    temp_rf_test_std = []\n",
        "\n",
        "    # Iterate through the AD-CTL, AD-MCI and MCI-CTL data\n",
        "    for j in range(3):\n",
        "      data_dir_1 = data_dir + dir_files[j]\n",
        "      file_1 = mat_files[j] + str(i) + '.mat'\n",
        "      mat_fname_1 = pjoin(data_dir_1, file_1)\n",
        "      mat_1 = sio.loadmat(mat_fname_1)\n",
        "      mat_1_train = mat_1['trainingSpread']\n",
        "      mat_1_test = mat_1['testSpread']\n",
        "\n",
        "      x_train_1 = mat_1_train[:, :-1]\n",
        "      y_train_1 = mat_1_train[:, -1]\n",
        "      x_test_1 = mat_1_test[:, :-1]\n",
        "      y_test_1 = mat_1_test[:, -1]\n",
        "      X_train = x_train_1\n",
        "      X_test = x_test_1\n",
        "\n",
        "      # Implement recursive feature elimination using the SVM linear kernel to get the topmost discriminative features\n",
        "      svm = SVR(kernel='linear', gamma='scale')\n",
        "      selector = RFE(svm, 900)\n",
        "      selector = selector.fit(x_train_1, y_train_1)\n",
        "      ft_ind = selector.get_support(1)\n",
        "      final_X_train = x_train_1[:,ft_ind]\n",
        "      final_X_test = x_test_1[:,ft_ind]\n",
        "      x_train_1 = final_X_train\n",
        "      x_test_1 = final_X_test\n",
        "\n",
        "      # Combine output of RFE - SVM with each feature's covariance matrix\n",
        "      cov_train_f = np.zeros((len(x_train_1),1))\n",
        "      for j in range(900):\n",
        "          cov_train = np.zeros((len(x_train_1),1))\n",
        "          cov_val = np.cov(x_train_1[:,j])\n",
        "          temp = []\n",
        "          for i in range(len(x_train_1)):\n",
        "              cov_train[i] = x_train_1[:,j][i] + cov_val\n",
        "          cov_train_f = np.column_stack((cov_train_f,cov_train))\n",
        "          final_cov_train = np.delete(cov_train_f, 0, axis=1)\n",
        "\n",
        "      cov_test_f = np.zeros((len(x_test_1),1))\n",
        "      for j in range(900):\n",
        "          cov_test = np.zeros((len(x_test_1),1))\n",
        "          cov_val = np.cov(x_test_1[:,j])\n",
        "          temp = []\n",
        "          for i in range(len(x_test_1)):\n",
        "              cov_test[i] = x_test_1[:,j][i] + cov_val\n",
        "          cov_test_f = np.column_stack((cov_test_f,cov_test))\n",
        "          final_cov_test = np.delete(cov_test_f, 0, axis=1)    \n",
        "\n",
        "      # Implement sequential forward selection on these features combined with each covariance matrix using the SVM linear kernel to get \n",
        "      # the 800 highly discriminative features\n",
        "      sfs = SFS(SVC(gamma='auto'), k_features=800, forward=True, floating=False, scoring = 'accuracy', cv = 0)\n",
        "      sfs.fit(final_cov_train, y_train_1)\n",
        "      sfs_ind = sfs.k_feature_idx_\n",
        "      final_X_train = x_train_1[:,sfs_ind] \n",
        "\n",
        "      sfs.fit(final_cov_test, y_test_1)\n",
        "      sfs_ind = sfs.k_feature_idx_\n",
        "      final_X_test = x_test_1[:,sfs_ind] \n",
        "\n",
        "      X_train = final_X_train\n",
        "      X_test = final_X_test\n",
        "\n",
        "      # Implement Random forest classifier\n",
        "\n",
        "      model = RandomForestClassifier()\n",
        "\n",
        "      model.fit(X_train, y_train_1)\n",
        "    \n",
        "      # Compute test accuracy\n",
        "      ypred_test = model.predict(X_test)\n",
        "      acc_test = accuracy_score(y_true=y_test_1, y_pred=ypred_test)\n",
        "      print('Test Accuracy: {:.4f}'.format(acc_test))\n",
        "      # Compute test specificity\n",
        "      temp_rf_test_acc.append(acc_test)\n",
        "      tn, fp, fn, tp = confusion_matrix(y_test_1, ypred_test).ravel()\n",
        "      FN_rf.append(fn)\n",
        "      FP_rf.append(fp)\n",
        "      TN_rf.append(tn)\n",
        "      TP_rf.append(tp)\n",
        "      print(tn,fp,fn,tp)\n",
        "      specificity_test = tn / (tn+fp)\n",
        "      temp_rf_test_spec.append(specificity_test)\n",
        "      print('Test Specificity: {:.4f}'.format(specificity_test))\n",
        "      # Compute test sensitivity\n",
        "      sensitivity_test = tp/ (tp+fn)\n",
        "      temp_rf_test_sens.append(sensitivity_test)\n",
        "      print('Test Sensitivity: {:.4f}'.format(sensitivity_test))\n",
        "      # Compute test mean absolute error\n",
        "      err_test = mean_absolute_error(y_test_1, ypred_test)\n",
        "      temp_rf_test_err.append(err_test)\n",
        "      print('Mean abs error: {:.4f}'.format(err_test))\n",
        "      # Compute test F1 score\n",
        "      p = tp/(tp+fp)\n",
        "      r = tp/(tp+fn)\n",
        "      f1_test = (2*p*r)/(r+p)\n",
        "      temp_rf_test_f1.append(f1_test)\n",
        "      print('F1 score: {:.4f}'.format(f1_test))\n",
        "      \n",
        "    # Append all the values for each split into the respective final accuracy, sensitivity, specificity, mean absolute error and F1 scores lists \n",
        "    final_rf_test_acc.append(temp_rf_test_acc)\n",
        "    final_rf_test_sens.append(temp_rf_test_sens)\n",
        "    final_rf_test_spec.append(temp_rf_test_spec)\n",
        "    final_rf_test_err.append(temp_rf_test_err)\n",
        "    final_rf_test_f1.append(temp_rf_test_f1)\n",
        "    final_rf_test_std.append(temp_rf_test_std)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YfnP6eJxdGGM",
        "colab_type": "text"
      },
      "source": [
        "*   **Implementing the Bagging ensemble of LDA classifiers**\n",
        "*   **Evaluating the train and test accuracy, sensitivity, specificity, mean absolute error, F1 score and confusion matrices of this classifier on the AD-CTL, AD-MCI and MCI-CTL data**\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ou6f438idJpM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.ensemble import BaggingClassifier\n",
        "# Create lists to store the final accuracies, specificity, sensitivity, mean absolute error, F1 scores and confusion matrices for each classifier\n",
        "final_bagging_test_acc = []\n",
        "final_bagging_test_sens = []\n",
        "final_bagging_test_spec = []\n",
        "final_bagging_test_err = []\n",
        "final_bagging_test_f1 = []\n",
        "TN_bag = []\n",
        "FN_bag = []\n",
        "TP_bag = []\n",
        "FP_bag = []\n",
        "dir_files = ['/AD_CTL', '/AD_MCI', '/MCI_CTL']\n",
        "mat_files = ['AD.CTL.0.aging.lookupFeatures.1000.perm1.split', 'AD.MCI.0.aging.lookupFeatures.1000.perm1.split', \n",
        "             'MCI.CTL.0.aging.lookupFeatures.1000.perm1.split']\n",
        "# Iterate through each of the 40 cross validation splits              \n",
        "for i in range(2,42):\n",
        "    data_dir = data_base_dir + '/SPLIT_' + str(i) \n",
        "    # Create lists to store the accuracies, specificity, sensitivity, mean absolute error, F1 scores and confusion matrices for each data split\n",
        "    temp_bagging_test_acc = []\n",
        "    temp_bagging_test_sens = []\n",
        "    temp_bagging_test_spec = []\n",
        "    temp_bagging_test_err = []\n",
        "    temp_bagging_test_f1 = []\n",
        " \n",
        "    # Iterate through the AD-CTL, AD-MCI and MCI-CTL data\n",
        "    for j in range(3):\n",
        "      data_dir_1 = data_dir + dir_files[j]\n",
        "      file_1 = mat_files[j] + str(i) + '.mat'\n",
        "      mat_fname_1 = pjoin(data_dir_1, file_1)\n",
        "      mat_1 = sio.loadmat(mat_fname_1)\n",
        "      mat_1_train = mat_1['trainingSpread']\n",
        "      mat_1_test = mat_1['testSpread']\n",
        "\n",
        "      x_train_1 = mat_1_train[:, :-1]\n",
        "      y_train_1 = mat_1_train[:, -1]\n",
        "      x_test_1 = mat_1_test[:, :-1]\n",
        "      y_test_1 = mat_1_test[:, -1]\n",
        "      X_train = x_train_1\n",
        "      X_test = x_test_1\n",
        "\n",
        "      # Implement recursive feature elimination using the SVM linear kernel to get the topmost discriminative features\n",
        "      svm = SVR(kernel='linear', gamma='scale')\n",
        "      selector = RFE(svm, 900)\n",
        "      selector = selector.fit(x_train_1, y_train_1)\n",
        "      ft_ind = selector.get_support(1)\n",
        "      final_X_train = x_train_1[:,ft_ind]\n",
        "      final_X_test = x_test_1[:,ft_ind]\n",
        "      x_train_1 = final_X_train\n",
        "      x_test_1 = final_X_test\n",
        "\n",
        "      # Combine output of RFE - SVM with each feature's covariance matrix\n",
        "      cov_train_f = np.zeros((len(x_train_1),1))\n",
        "      for j in range(900):\n",
        "          cov_train = np.zeros((len(x_train_1),1))\n",
        "          cov_val = np.cov(x_train_1[:,j])\n",
        "          temp = []\n",
        "          for i in range(len(x_train_1)):\n",
        "              cov_train[i] = x_train_1[:,j][i] + cov_val\n",
        "          cov_train_f = np.column_stack((cov_train_f,cov_train))\n",
        "          final_cov_train = np.delete(cov_train_f, 0, axis=1)\n",
        "\n",
        "      cov_test_f = np.zeros((len(x_test_1),1))\n",
        "      for j in range(900):\n",
        "          cov_test = np.zeros((len(x_test_1),1))\n",
        "          cov_val = np.cov(x_test_1[:,j])\n",
        "          temp = []\n",
        "          for i in range(len(x_test_1)):\n",
        "              cov_test[i] = x_test_1[:,j][i] + cov_val\n",
        "          cov_test_f = np.column_stack((cov_test_f,cov_test))\n",
        "          final_cov_test = np.delete(cov_test_f, 0, axis=1)    \n",
        "\n",
        "      # Implement sequential forward selection on these features combined with each covariance matrix using the SVM linear kernel to get \n",
        "      # the 800 highly discriminative features\n",
        "      sfs = SFS(SVC(gamma='auto'), k_features=800, forward=True, floating=False, scoring = 'accuracy', cv = 0)\n",
        "      sfs.fit(final_cov_train, y_train_1)\n",
        "      sfs_ind = sfs.k_feature_idx_\n",
        "      final_X_train = x_train_1[:,sfs_ind] \n",
        "\n",
        "      sfs.fit(final_cov_test, y_test_1)\n",
        "      sfs_ind = sfs.k_feature_idx_\n",
        "      final_X_test = x_test_1[:,sfs_ind] \n",
        "\n",
        "      X_train = final_X_train\n",
        "      X_test = final_X_test\n",
        "\n",
        "      # Implement bagging ensemble of LDA classifiers\n",
        "\n",
        "      ensemble = BaggingClassifier(LDA()) \n",
        "      ensemble.fit(X_train, y_train_1)\n",
        "    \n",
        "      # Compute test accuracy\n",
        "      ypred_test = ensemble.predict(X_test)\n",
        "      acc_test = accuracy_score(y_true=y_test_1, y_pred=ypred_test)\n",
        "      print('Test Accuracy: {:.4f}'.format(acc_test))\n",
        "      temp_bagging_test_acc.append(acc_test)\n",
        "      # Compute test specificity\n",
        "      tn, fp, fn, tp = confusion_matrix(y_test_1, ypred_test).ravel()\n",
        "      TN_bag.append(tn)\n",
        "      TP_bag.append(tp)\n",
        "      FN_bag.append(fn)\n",
        "      FP_bag.append(fp)\n",
        "      print(tn,fp,fn,tp)\n",
        "      specificity_test = tn / (tn+fp)\n",
        "      temp_bagging_test_spec.append(specificity_test)\n",
        "      print('Test Specificity: {:.4f}'.format(specificity_test))\n",
        "      # Compute test sensitivity\n",
        "      sensitivity_test = tp/ (tp+fn)\n",
        "      temp_bagging_test_sens.append(sensitivity_test)\n",
        "      print('Test Sensitivity: {:.4f}'.format(sensitivity_test))\n",
        "      # Compute test mean absolute error\n",
        "      err_test = mean_absolute_error(y_test_1, ypred_test)\n",
        "      temp_bagging_test_err.append(err_test)\n",
        "      print('Mean abs error: {:.4f}'.format(err_test))\n",
        "      # Compute test F1 score\n",
        "      p = tp/(tp+fp)\n",
        "      r = tp/(tp+fn)\n",
        "      f1_test = (2*p*r)/(r+p)\n",
        "      temp_bagging_test_f1.append(f1_test)\n",
        "      print('F1 score: {:.4f}'.format(f1_test))\n",
        "    \n",
        "    # Append all the values for each split into the respective final accuracy, sensitivity, specificity, mean absolute error and F1 scores lists \n",
        "    final_bagging_test_acc.append(temp_bagging_test_acc)\n",
        "    final_bagging_test_sens.append(temp_bagging_test_sens)\n",
        "    final_bagging_test_spec.append(temp_bagging_test_spec)\n",
        "    final_bagging_test_err.append(temp_bagging_test_err)\n",
        "    final_bagging_test_f1.append(temp_bagging_test_f1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WnDOvA5JdLyp",
        "colab_type": "text"
      },
      "source": [
        "*   **Implementing the Naive Bayes classifier**\n",
        "*   **Evaluating the train and test accuracy, sensitivity, specificity, mean absolute error, F1 score and confusion matrices of this classifier on the AD-CTL, AD-MCI and MCI-CTL data**\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C49K3UbwdOc8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.naive_bayes import GaussianNB \n",
        "# Create lists to store the final accuracies, specificity, sensitivity, mean absolute error, F1 scores and confusion matrices for each classifier\n",
        "final_naivebayes_test_acc = []\n",
        "final_naivebayes_test_sens = []\n",
        "final_naivebayes_test_spec = []\n",
        "final_naivebayes_test_err = []\n",
        "final_naivebayes_test_f1 = []\n",
        "TN_nb = []\n",
        "FN_nb = []\n",
        "FP_nb = []\n",
        "TP_nb = []\n",
        "dir_files = ['/AD_CTL', '/AD_MCI', '/MCI_CTL']\n",
        "mat_files = ['AD.CTL.0.aging.lookupFeatures.1000.perm1.split', 'AD.MCI.0.aging.lookupFeatures.1000.perm1.split', \n",
        "             'MCI.CTL.0.aging.lookupFeatures.1000.perm1.split']\n",
        "# Iterate through each of the 40 cross validation splits              \n",
        "for i in range(2,42):\n",
        "    data_dir = data_base_dir + '/SPLIT_' + str(i) \n",
        "    # Create lists to store the accuracies, specificity, sensitivity, mean absolute error, F1 scores and confusion matrices for each data split\n",
        "    temp_naivebayes_test_acc = []\n",
        "    temp_naivebayes_test_sens = []\n",
        "    temp_naivebayes_test_spec = []\n",
        "    temp_naivebayes_test_err = []\n",
        "    temp_naivebayes_test_f1 = []\n",
        " \n",
        "    # Iterate through the AD-CTL, AD-MCI and MCI-CTL data\n",
        "    for j in range(3):\n",
        "      data_dir_1 = data_dir + dir_files[j]\n",
        "      file_1 = mat_files[j] + str(i) + '.mat'\n",
        "      mat_fname_1 = pjoin(data_dir_1, file_1)\n",
        "      mat_1 = sio.loadmat(mat_fname_1)\n",
        "      mat_1_train = mat_1['trainingSpread']\n",
        "      mat_1_test = mat_1['testSpread']\n",
        "\n",
        "      x_train_1 = mat_1_train[:, :-1]\n",
        "      y_train_1 = mat_1_train[:, -1]\n",
        "      x_test_1 = mat_1_test[:, :-1]\n",
        "      y_test_1 = mat_1_test[:, -1]\n",
        "      X_train = x_train_1\n",
        "      X_test = x_test_1\n",
        "\n",
        "      # Implement recursive feature elimination using the SVM linear kernel to get the topmost discriminative features\n",
        "      svm = SVR(kernel='linear', gamma='scale')\n",
        "      selector = RFE(svm, 900)\n",
        "      selector = selector.fit(x_train_1, y_train_1)\n",
        "      ft_ind = selector.get_support(1)\n",
        "      final_X_train = x_train_1[:,ft_ind]\n",
        "      final_X_test = x_test_1[:,ft_ind]\n",
        "      x_train_1 = final_X_train\n",
        "      x_test_1 = final_X_test\n",
        "\n",
        "      # Combine output of RFE - SVM with each feature's covariance matrix\n",
        "      cov_train_f = np.zeros((len(x_train_1),1))\n",
        "      for j in range(900):\n",
        "          cov_train = np.zeros((len(x_train_1),1))\n",
        "          cov_val = np.cov(x_train_1[:,j])\n",
        "          temp = []\n",
        "          for i in range(len(x_train_1)):\n",
        "              cov_train[i] = x_train_1[:,j][i] + cov_val\n",
        "          cov_train_f = np.column_stack((cov_train_f,cov_train))\n",
        "          final_cov_train = np.delete(cov_train_f, 0, axis=1)\n",
        "\n",
        "      cov_test_f = np.zeros((len(x_test_1),1))\n",
        "      for j in range(900):\n",
        "          cov_test = np.zeros((len(x_test_1),1))\n",
        "          cov_val = np.cov(x_test_1[:,j])\n",
        "          temp = []\n",
        "          for i in range(len(x_test_1)):\n",
        "              cov_test[i] = x_test_1[:,j][i] + cov_val\n",
        "          cov_test_f = np.column_stack((cov_test_f,cov_test))\n",
        "          final_cov_test = np.delete(cov_test_f, 0, axis=1)    \n",
        "\n",
        "      # Implement sequential forward selection on these features combined with each covariance matrix using the SVM linear kernel to get \n",
        "      # the 800 highly discriminative features\n",
        "      sfs = SFS(SVC(gamma='auto'), k_features=800, forward=True, floating=False, scoring = 'accuracy', cv = 0)\n",
        "      sfs.fit(final_cov_train, y_train_1)\n",
        "      sfs_ind = sfs.k_feature_idx_\n",
        "      final_X_train = x_train_1[:,sfs_ind] \n",
        "\n",
        "      sfs.fit(final_cov_test, y_test_1)\n",
        "      sfs_ind = sfs.k_feature_idx_\n",
        "      final_X_test = x_test_1[:,sfs_ind] \n",
        "\n",
        "      X_train = final_X_train\n",
        "      X_test = final_X_test\n",
        "\n",
        "      # Implement Naive Bayes classifier\n",
        "\n",
        "      ensemble = GaussianNB() \n",
        "      ensemble.fit(X_train, y_train_1)\n",
        "    \n",
        "      # Compute test accuracy\n",
        "      ypred_test = ensemble.predict(X_test)\n",
        "      acc_test = accuracy_score(y_true=y_test_1, y_pred=ypred_test)\n",
        "      print('Test Accuracy: {:.4f}'.format(acc_test))\n",
        "      # Compute test specificity\n",
        "      temp_naivebayes_test_acc.append(acc_test)\n",
        "      tn, fp, fn, tp = confusion_matrix(y_test_1, ypred_test).ravel()\n",
        "      TN_nb.append(tn)\n",
        "      TP_nb.append(tp)\n",
        "      FP_nb.append(fp)\n",
        "      FN_nb.append(fn)\n",
        "      print(tn,fp,fn,tp)\n",
        "      specificity_test = tn / (tn+fp)\n",
        "      temp_naivebayes_test_spec.append(specificity_test)\n",
        "      print('Test Specificity: {:.4f}'.format(specificity_test))\n",
        "      # Compute test sensitivity\n",
        "      sensitivity_test = tp/ (tp+fn)\n",
        "      temp_naivebayes_test_sens.append(sensitivity_test)\n",
        "      print('Test Sensitivity: {:.4f}'.format(sensitivity_test))\n",
        "      # Compute test mean absolute error\n",
        "      err_test = mean_absolute_error(y_test_1, ypred_test)\n",
        "      temp_naivebayes_test_err.append(err_test)\n",
        "      print('Mean abs error: {:.4f}'.format(err_test))\n",
        "      # Compute test F1 score\n",
        "      p = tp/(tp+fp)\n",
        "      r = tp/(tp+fn)\n",
        "      f1_test = (2*p*r)/(r+p)\n",
        "      temp_naivebayes_test_f1.append(f1_test)\n",
        "      print('F1 score: {:.4f}'.format(f1_test))\n",
        "    \n",
        "    # Append all the values for each split into the respective final accuracy, sensitivity, specificity, mean absolute error and F1 scores lists \n",
        "    final_naivebayes_test_acc.append(temp_naivebayes_test_acc)\n",
        "    final_naivebayes_test_sens.append(temp_naivebayes_test_sens)\n",
        "    final_naivebayes_test_spec.append(temp_naivebayes_test_spec)\n",
        "    final_naivebayes_test_err.append(temp_naivebayes_test_err)\n",
        "    final_naivebayes_test_f1.append(temp_naivebayes_test_f1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1Qi8FsDOdQj1",
        "colab_type": "text"
      },
      "source": [
        "**Compute mean and standard deviation of test accuracy for each of these classifiers**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MNLhxryOdUAR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# LDA classifier\n",
        "AC_LDA_TA = []\n",
        "AM_LDA_TA = []\n",
        "MC_LDA_TA = []\n",
        "for i in range(len(final_LDA_test_acc)):\n",
        "      AC_LDA_TA.append(final_LDA_test_acc[i][0]*100)\n",
        "      AM_LDA_TA.append(final_LDA_test_acc[i][1]*100)\n",
        "      MC_LDA_TA.append(final_LDA_test_acc[i][2]*100)\n",
        "# Compute mean LDA test accuracies for each AD-CTL, AD-MCI and MCI-CTL binary classification problem         \n",
        "Mean_AC_LDA_TA = sum(AC_LDA_TA)/len(AC_LDA_TA)\n",
        "Mean_AM_LDA_TA = sum(AM_LDA_TA)/len(AM_LDA_TA)\n",
        "Mean_MC_LDA_TA = sum(MC_LDA_TA)/len(MC_LDA_TA)\n",
        "print('Mean_AC_LDA_TA', Mean_AC_LDA_TA)\n",
        "print('Mean_AM_LDA_TA', Mean_AM_LDA_TA)\n",
        "print('Mean_MC_LDA_TA', Mean_MC_LDA_TA)\n",
        "# Compute standard deviation of the LDA test accuracies for each AD-CTL, AD-MCI and MCI-CTL binary classification problem    \n",
        "print('Std_AC_LDA_TA',statistics.stdev(AC_LDA_TA))\n",
        "print('Std_AM_LDA_TA',statistics.stdev(AM_LDA_TA))\n",
        "print('Std_MC_LDA_TA',statistics.stdev(MC_LDA_TA))\n",
        "\n",
        "# Logistic regression classifier\n",
        "AC_SGD_TA = []\n",
        "AM_SGD_TA = []\n",
        "MC_SGD_TA = []\n",
        "for i in range(len(final_SGD_test_acc)):\n",
        "      AC_SGD_TA.append(final_SGD_test_acc[i][0]*100)\n",
        "      AM_SGD_TA.append(final_SGD_test_acc[i][1]*100)\n",
        "      MC_SGD_TA.append(final_SGD_test_acc[i][2]*100)\n",
        "# Compute mean logistic regression test accuracies for each AD-CTL, AD-MCI and MCI-CTL binary classification problem          \n",
        "Mean_AC_SGD_TA = sum(AC_SGD_TA)/len(AC_SGD_TA)\n",
        "Mean_AM_SGD_TA = sum(AM_SGD_TA)/len(AM_SGD_TA)\n",
        "Mean_MC_SGD_TA = sum(MC_SGD_TA)/len(MC_SGD_TA)\n",
        "print('Mean_AC_SGD_TA', Mean_AC_SGD_TA)\n",
        "print('Mean_AM_SGD_TA', Mean_AM_SGD_TA)\n",
        "print('Mean_MC_SGD_TA', Mean_MC_SGD_TA)\n",
        "# Compute standard deviation of the logistic regression test accuracies for each AD-CTL, AD-MCI and MCI-CTL binary classification problem    \n",
        "print('Std_AC_SGD_TA',statistics.stdev(AC_SGD_TA))\n",
        "print('Std_AM_SGD_TA',statistics.stdev(AM_SGD_TA))\n",
        "print('Std_MC_SGD_TA',statistics.stdev(MC_SGD_TA))\n",
        "\n",
        "# SVM classifier\n",
        "AC_SVM_TA = []\n",
        "AM_SVM_TA = []\n",
        "MC_SVM_TA = []\n",
        "for i in range(len(final_SVM_test_acc)):\n",
        "      AC_SVM_TA.append(final_SVM_test_acc[i][0]*100)\n",
        "      AM_SVM_TA.append(final_SVM_test_acc[i][1]*100)\n",
        "      MC_SVM_TA.append(final_SVM_test_acc[i][2]*100)\n",
        "# Compute mean SVM test accuracies for each AD-CTL, AD-MCI and MCI-CTL binary classification problem          \n",
        "Mean_AC_SVM_TA = sum(AC_SVM_TA)/len(AC_SVM_TA)\n",
        "Mean_AM_SVM_TA = sum(AM_SVM_TA)/len(AM_SVM_TA)\n",
        "Mean_MC_SVM_TA = sum(MC_SVM_TA)/len(MC_SVM_TA)\n",
        "print('Mean_AC_SVM_TA', Mean_AC_SVM_TA)\n",
        "print('Mean_AM_SVM_TA', Mean_AM_SVM_TA)\n",
        "print('Mean_MC_SVM_TA', Mean_MC_SVM_TA)\n",
        "# Compute standard deviation of the SVM test accuracies for each AD-CTL, AD-MCI and MCI-CTL binary classification problem    \n",
        "print('Std_AC_SVM_TA',statistics.stdev(AC_SVM_TA))\n",
        "print('Std_AM_SVM_TA',statistics.stdev(AM_SVM_TA))\n",
        "print('Std_MC_SVM_TA',statistics.stdev(MC_SVM_TA))\n",
        "\n",
        "# Decision tree classifier\n",
        "AC_DT_TA = []\n",
        "AM_DT_TA = []\n",
        "MC_DT_TA = []\n",
        "for i in range(len(final_DT_test_acc)):\n",
        "      AC_DT_TA.append(final_DT_test_acc[i][0]*100)\n",
        "      AM_DT_TA.append(final_DT_test_acc[i][1]*100)\n",
        "      MC_DT_TA.append(final_DT_test_acc[i][2]*100)\n",
        "# Compute mean decision tree test accuracies for each AD-CTL, AD-MCI and MCI-CTL binary classification problem          \n",
        "Mean_AC_DT_TA = sum(AC_DT_TA)/len(AC_DT_TA)\n",
        "Mean_AM_DT_TA = sum(AM_DT_TA)/len(AM_DT_TA)\n",
        "Mean_MC_DT_TA = sum(MC_DT_TA)/len(MC_DT_TA)\n",
        "print('Mean_AC_DT_TA', Mean_AC_DT_TA)\n",
        "print('Mean_AM_DT_TA', Mean_AM_DT_TA)\n",
        "print('Mean_MC_DT_TA', Mean_MC_DT_TA)\n",
        "# Compute standard deviation of the decision tree test accuracies for each AD-CTL, AD-MCI and MCI-CTL binary classification problem    \n",
        "print('Std_AC_DT_TA',statistics.stdev(AC_DT_TA))\n",
        "print('Std_AM_DT_TA',statistics.stdev(AM_DT_TA))\n",
        "print('Std_MC_DT_TA',statistics.stdev(MC_DT_TA))\n",
        "\n",
        "# KNN classifier\n",
        "AC_KNN_TA = []\n",
        "AM_KNN_TA = []\n",
        "MC_KNN_TA = []\n",
        "for i in range(len(final_KNN_test_acc)):\n",
        "      AC_KNN_TA.append(final_KNN_test_acc[i][0]*100)\n",
        "      AM_KNN_TA.append(final_KNN_test_acc[i][1]*100)\n",
        "      MC_KNN_TA.append(final_KNN_test_acc[i][2]*100)\n",
        "# Compute mean KNN test accuracies for each AD-CTL, AD-MCI and MCI-CTL binary classification problem          \n",
        "Mean_AC_KNN_TA = sum(AC_KNN_TA)/len(AC_KNN_TA)\n",
        "Mean_AM_KNN_TA = sum(AM_KNN_TA)/len(AM_KNN_TA)\n",
        "Mean_MC_KNN_TA = sum(MC_KNN_TA)/len(MC_KNN_TA) \n",
        "print('Mean_AC_KNN_TA', Mean_AC_KNN_TA)\n",
        "print('Mean_AM_KNN_TA', Mean_AM_KNN_TA)\n",
        "print('Mean_MC_KNN_TA', Mean_MC_KNN_TA)\n",
        "# Compute standard deviation of the KNN test accuracies for each AD-CTL, AD-MCI and MCI-CTL binary classification problem    \n",
        "print('Std_AC_KNN_TA',statistics.stdev(AC_KNN_TA))\n",
        "print('Std_AM_KNN_TA',statistics.stdev(AM_KNN_TA))\n",
        "print('Std_MC_KNN_TA',statistics.stdev(MC_KNN_TA))\n",
        "\n",
        "# Stacking ensemble of classifiers\n",
        "AC_stack_TA = []\n",
        "AM_stack_TA = []\n",
        "MC_stack_TA = []\n",
        "for i in range(len(final_stack_test_acc)):\n",
        "      AC_stack_TA.append(final_stack_test_acc[i][0]*100)\n",
        "      AM_stack_TA.append(final_stack_test_acc[i][1]*100)\n",
        "      MC_stack_TA.append(final_stack_test_acc[i][2]*100)\n",
        "# Compute mean stacking ensemble test accuracies for each AD-CTL, AD-MCI and MCI-CTL binary classification problem          \n",
        "Mean_AC_stack_TA = sum(AC_stack_TA)/len(AC_stack_TA)\n",
        "Mean_AM_stack_TA = sum(AM_stack_TA)/len(AM_stack_TA)\n",
        "Mean_MC_stack_TA = sum(MC_stack_TA)/len(MC_stack_TA)\n",
        "print('Mean_AC_stack_TA', Mean_AC_stack_TA)\n",
        "print('Mean_AM_stack_TA', Mean_AM_stack_TA)\n",
        "print('Mean_MC_stack_TA', Mean_MC_stack_TA)\n",
        "# Compute standard deviation of the stacking ensemble test accuracies for each AD-CTL, AD-MCI and MCI-CTL binary classification problem    \n",
        "print('Std_AC_stack_TA',statistics.stdev(AC_stack_TA))\n",
        "print('Std_AM_stack_TA',statistics.stdev(AM_stack_TA))\n",
        "print('Std_MC_stack_TA',statistics.stdev(MC_stack_TA))\n",
        "\n",
        "# Random forest classifier\n",
        "AC_rf_TA = []\n",
        "AM_rf_TA = []\n",
        "MC_rf_TA = []\n",
        "for i in range(len(final_rf_test_acc)):\n",
        "      AC_rf_TA.append(final_rf_test_acc[i][0]*100)\n",
        "      AM_rf_TA.append(final_rf_test_acc[i][1]*100)\n",
        "      MC_rf_TA.append(final_rf_test_acc[i][2]*100)\n",
        "# Compute mean random forest test accuracies for each AD-CTL, AD-MCI and MCI-CTL binary classification problem          \n",
        "Mean_AC_rf_TA = sum(AC_rf_TA)/len(AC_rf_TA)\n",
        "Mean_AM_rf_TA = sum(AM_rf_TA)/len(AM_rf_TA)\n",
        "Mean_MC_rf_TA = sum(MC_rf_TA)/len(MC_rf_TA)\n",
        "print('Mean_AC_rf_TA', Mean_AC_rf_TA)\n",
        "print('Mean_AM_rf_TA', Mean_AM_rf_TA)\n",
        "print('Mean_MC_rf_TA', Mean_MC_rf_TA)\n",
        "# Compute standard deviation of the random forest test accuracies for each AD-CTL, AD-MCI and MCI-CTL binary classification problem    \n",
        "print('Std_AC_rf_TA',statistics.stdev(AC_rf_TA))\n",
        "print('Std_AM_rf_TA',statistics.stdev(AM_rf_TA))\n",
        "print('Std_MC_rf_TA',statistics.stdev(MC_rf_TA))\n",
        "\n",
        "# Bagging ensemble of classifiers\n",
        "AC_bagging_TA = []\n",
        "AM_bagging_TA = []\n",
        "MC_bagging_TA = []\n",
        "for i in range(len(final_bagging_test_acc)):\n",
        "      AC_bagging_TA.append(final_bagging_test_acc[i][0]*100)\n",
        "      AM_bagging_TA.append(final_bagging_test_acc[i][1]*100)\n",
        "      MC_bagging_TA.append(final_bagging_test_acc[i][2]*100)\n",
        "# Compute mean bagging ensemble accuracies for each AD-CTL, AD-MCI and MCI-CTL binary classification problem             \n",
        "Mean_AC_bagging_TA = sum(AC_bagging_TA)/len(AC_bagging_TA)\n",
        "Mean_AM_bagging_TA = sum(AM_bagging_TA)/len(AM_bagging_TA)\n",
        "Mean_MC_bagging_TA = sum(MC_bagging_TA)/len(MC_bagging_TA)\n",
        "print('Mean_AC_bagging_TA', Mean_AC_bagging_TA)\n",
        "print('Mean_AM_bagging_TA', Mean_AM_bagging_TA)\n",
        "print('Mean_MC_bagging_TA', Mean_MC_bagging_TA)\n",
        "# Compute standard deviation of the bagging ensemble test accuracies for each AD-CTL, AD-MCI and MCI-CTL binary classification problem    \n",
        "print('Std_AC_bagging_TA',statistics.stdev(AC_bagging_TA))\n",
        "print('Std_AM_bagging_TA',statistics.stdev(AM_bagging_TA))\n",
        "print('Std_MC_bagging_TA',statistics.stdev(MC_bagging_TA))\n",
        "\n",
        "# Naive bayes classifier\n",
        "AC_naivebayes_TA = []\n",
        "AM_naivebayes_TA = []\n",
        "MC_naivebayes_TA = []\n",
        "for i in range(len(final_naivebayes_test_acc)):\n",
        "      AC_naivebayes_TA.append(final_naivebayes_test_acc[i][0]*100)\n",
        "      AM_naivebayes_TA.append(final_naivebayes_test_acc[i][1]*100)\n",
        "      MC_naivebayes_TA.append(final_naivebayes_test_acc[i][2]*100)\n",
        "# Compute mean naive bayes accuracies for each AD-CTL, AD-MCI and MCI-CTL binary classification problem             \n",
        "Mean_AC_naivebayes_TA = sum(AC_naivebayes_TA)/len(AC_naivebayes_TA)\n",
        "Mean_AM_naivebayes_TA = sum(AM_naivebayes_TA)/len(AM_naivebayes_TA)\n",
        "Mean_MC_naivebayes_TA = sum(MC_naivebayes_TA)/len(MC_naivebayes_TA)\n",
        "print('Mean_AC_naivebayes_TA', Mean_AC_naivebayes_TA)\n",
        "print('Mean_AM_naivebayes_TA', Mean_AM_naivebayes_TA)\n",
        "print('Mean_MC_naivebayes_TA', Mean_MC_naivebayes_TA)\n",
        "# Compute standard deviation of the naive bayes test accuracies for each AD-CTL, AD-MCI and MCI-CTL binary classification problem    \n",
        "print('Std_AC_naivebayes_TA',statistics.stdev(AC_naivebayes_TA))\n",
        "print('Std_AM_naivebayes_TA',statistics.stdev(AM_naivebayes_TA))\n",
        "print('Std_MC_naivebayes_TA',statistics.stdev(MC_naivebayes_TA))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Is6vRL85dWIv",
        "colab_type": "text"
      },
      "source": [
        "**Compute mean and standard deviation of test sensitivity for each of these classifiers**\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ysRioslYdY9L",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# LDA classifier\n",
        "AC_LDA_TSens = []\n",
        "AM_LDA_TSens = []\n",
        "MC_LDA_TSens = []\n",
        "for i in range(len(final_LDA_test_sens)):\n",
        "      AC_LDA_TSens.append(final_LDA_test_sens[i][0]*100)\n",
        "      AM_LDA_TSens.append(final_LDA_test_sens[i][1]*100)\n",
        "      MC_LDA_TSens.append(final_LDA_test_sens[i][2]*100)\n",
        "# Compute mean LDA test sensitivity for each AD-CTL, AD-MCI and MCI-CTL binary classification problem              \n",
        "Mean_AC_LDA_TSens = sum(AC_LDA_TSens)/len(AC_LDA_TSens)\n",
        "Mean_AM_LDA_TSens = sum(AM_LDA_TSens)/len(AM_LDA_TSens)\n",
        "Mean_MC_LDA_TSens = sum(MC_LDA_TSens)/len(MC_LDA_TSens) \n",
        "print('Mean_AC_LDA_TSens', Mean_AC_LDA_TSens)\n",
        "print('Mean_AM_LDA_TSens', Mean_AM_LDA_TSens)\n",
        "print('Mean_MC_LDA_TSens', Mean_MC_LDA_TSens)  \n",
        "# Compute standard deviation of the LDA test sensitivity for each AD-CTL, AD-MCI and MCI-CTL binary classification problem    \n",
        "print('Std_AC_LDA_TSens', statistics.stdev(AC_LDA_TSens))\n",
        "print('Std_AM_LDA_TSens', statistics.stdev(AM_LDA_TSens))\n",
        "print('Std_MC_LDA_TSens', statistics.stdev(MC_LDA_TSens))\n",
        "\n",
        "# Logistic regression classifier\n",
        "AC_SGD_TSens = []\n",
        "AM_SGD_TSens = []\n",
        "MC_SGD_TSens = []\n",
        "for i in range(len(final_SGD_test_sens)):\n",
        "      AC_SGD_TSens.append(final_SGD_test_sens[i][0]*100)\n",
        "      AM_SGD_TSens.append(final_SGD_test_sens[i][1]*100)\n",
        "      MC_SGD_TSens.append(final_SGD_test_sens[i][2]*100)\n",
        "# Compute mean logistic regression test sensitivity for each AD-CTL, AD-MCI and MCI-CTL binary classification problem              \n",
        "Mean_AC_SGD_TSens = sum(AC_SGD_TSens)/len(AC_SGD_TSens)\n",
        "Mean_AM_SGD_TSens = sum(AM_SGD_TSens)/len(AM_SGD_TSens)\n",
        "Mean_MC_SGD_TSens = sum(MC_SGD_TSens)/len(MC_SGD_TSens) \n",
        "print('Mean_AC_SGD_TSens', Mean_AC_SGD_TSens)\n",
        "print('Mean_AM_SGD_TSens', Mean_AM_SGD_TSens)\n",
        "print('Mean_MC_SGD_TSens', Mean_MC_SGD_TSens)  \n",
        "# Compute standard deviation of the logistic regression test sensitivity for each AD-CTL, AD-MCI and MCI-CTL binary classification problem    \n",
        "print('Std_AC_SGD_TSens', statistics.stdev(AC_SGD_TSens))\n",
        "print('Std_AM_SGD_TSens', statistics.stdev(AM_SGD_TSens))\n",
        "print('Std_MC_SGD_TSens', statistics.stdev(MC_SGD_TSens))\n",
        "\n",
        "# SVM classifier\n",
        "AC_SVM_TSens = []\n",
        "AM_SVM_TSens = []\n",
        "MC_SVM_TSens = []\n",
        "for i in range(len(final_SVM_test_sens)):\n",
        "      AC_SVM_TSens.append(final_SVM_test_sens[i][0]*100)\n",
        "      AM_SVM_TSens.append(final_SVM_test_sens[i][1]*100)\n",
        "      MC_SVM_TSens.append(final_SVM_test_sens[i][2]*100)\n",
        "# Compute mean SVM test sensitivity for each AD-CTL, AD-MCI and MCI-CTL binary classification problem              \n",
        "Mean_AC_SVM_TSens = sum(AC_SVM_TSens)/len(AC_SVM_TSens)\n",
        "Mean_AM_SVM_TSens = sum(AM_SVM_TSens)/len(AM_SVM_TSens)\n",
        "Mean_MC_SVM_TSens = sum(MC_SVM_TSens)/len(MC_SVM_TSens)   \n",
        "print('Mean_AC_SVM_TSens', Mean_AC_SVM_TSens)\n",
        "print('Mean_AM_SVM_TSens', Mean_AM_SVM_TSens)\n",
        "print('Mean_MC_SVM_TSens', Mean_MC_SVM_TSens)  \n",
        "# Compute standard deviation of the SVM test sensitivity for each AD-CTL, AD-MCI and MCI-CTL binary classification problem    \n",
        "print('Std_AC_SVM_TSens', statistics.stdev(AC_SVM_TSens))\n",
        "print('Std_AM_SVM_TSens', statistics.stdev(AM_SVM_TSens))\n",
        "print('Std_MC_SVM_TSens', statistics.stdev(MC_SVM_TSens))\n",
        "\n",
        "# Decision tree classifier\n",
        "AC_DT_TSens = []\n",
        "AM_DT_TSens = []\n",
        "MC_DT_TSens = []\n",
        "for i in range(len(final_DT_test_sens)):\n",
        "      AC_DT_TSens.append(final_DT_test_sens[i][0]*100)\n",
        "      AM_DT_TSens.append(final_DT_test_sens[i][1]*100)\n",
        "      MC_DT_TSens.append(final_DT_test_sens[i][2]*100)\n",
        "# Compute mean decision tree test sensitivity for each AD-CTL, AD-MCI and MCI-CTL binary classification problem              \n",
        "Mean_AC_DT_TSens = sum(AC_DT_TSens)/len(AC_DT_TSens)\n",
        "Mean_AM_DT_TSens = sum(AM_DT_TSens)/len(AM_DT_TSens)\n",
        "Mean_MC_DT_TSens = sum(MC_DT_TSens)/len(MC_DT_TSens)  \n",
        "print('Mean_AC_DT_TSens', Mean_AC_DT_TSens)\n",
        "print('Mean_AM_DT_TSens', Mean_AM_DT_TSens)\n",
        "print('Mean_MC_DT_TSens', Mean_MC_DT_TSens)  \n",
        "# Compute standard deviation of the decision tree test sensitivity for each AD-CTL, AD-MCI and MCI-CTL binary classification problem    \n",
        "print('Std_AC_DT_TSens', statistics.stdev(AC_DT_TSens))\n",
        "print('Std_AM_DT_TSens', statistics.stdev(AM_DT_TSens))\n",
        "print('Std_MC_DT_TSens', statistics.stdev(MC_DT_TSens))\n",
        "\n",
        "# KNN classifier\n",
        "AC_KNN_TSens = []\n",
        "AM_KNN_TSens = []\n",
        "MC_KNN_TSens = []\n",
        "for i in range(len(final_KNN_test_sens)):\n",
        "      AC_KNN_TSens.append(final_KNN_test_sens[i][0]*100)\n",
        "      AM_KNN_TSens.append(final_KNN_test_sens[i][1]*100)\n",
        "      MC_KNN_TSens.append(final_KNN_test_sens[i][2]*100)\n",
        "# Compute mean KNN test sensitivity for each AD-CTL, AD-MCI and MCI-CTL binary classification problem              \n",
        "Mean_AC_KNN_TSens = sum(AC_KNN_TSens)/len(AC_KNN_TSens)\n",
        "Mean_AM_KNN_TSens = sum(AM_KNN_TSens)/len(AM_KNN_TSens)\n",
        "Mean_MC_KNN_TSens = sum(MC_KNN_TSens)/len(MC_KNN_TSens) \n",
        "print('Mean_AC_KNN_TSens', Mean_AC_KNN_TSens)\n",
        "print('Mean_AM_KNN_TSens', Mean_AM_KNN_TSens)\n",
        "print('Mean_MC_KNN_TSens', Mean_MC_KNN_TSens)    \n",
        "# Compute standard deviation of the KNN test sensitivity for each AD-CTL, AD-MCI and MCI-CTL binary classification problem    \n",
        "print('Std_AC_KNN_TSens', statistics.stdev(AC_KNN_TSens))\n",
        "print('Std_AM_KNN_TSens', statistics.stdev(AM_KNN_TSens))\n",
        "print('Std_MC_KNN_TSens', statistics.stdev(MC_KNN_TSens))\n",
        "\n",
        "# Stacking ensemble of classifiers\n",
        "AC_stack_sens = []\n",
        "AM_stack_sens = []\n",
        "MC_stack_sens = []\n",
        "for i in range(len(final_stack_test_sens)):\n",
        "      AC_stack_sens.append(final_stack_test_sens[i][0]*100)\n",
        "      AM_stack_sens.append(final_stack_test_sens[i][1]*100)\n",
        "      MC_stack_sens.append(final_stack_test_sens[i][2]*100)\n",
        "# Compute mean stacking ensemble test sensitivity for each AD-CTL, AD-MCI and MCI-CTL binary classification problem                  \n",
        "Mean_AC_stack_sens = sum(AC_stack_sens)/len(AC_stack_sens)\n",
        "Mean_AM_stack_sens = sum(AM_stack_sens)/len(AM_stack_sens)\n",
        "Mean_MC_stack_sens = sum(MC_stack_sens)/len(MC_stack_sens)\n",
        "print('Mean_AC_stack_Tsens', Mean_AC_stack_sens)\n",
        "print('Mean_AM_stack_Tsens', Mean_AM_stack_sens)\n",
        "print('Mean_MC_stack_Tsens', Mean_MC_stack_sens)\n",
        "# Compute standard deviation of the stacking ensemble test sensitivity for each AD-CTL, AD-MCI and MCI-CTL binary classification problem    \n",
        "print('Std_AC_stack_Tsens',statistics.stdev(AC_stack_sens))\n",
        "print('Std_AM_stack_Tsens',statistics.stdev(AM_stack_sens))\n",
        "print('Std_MC_stack_Tsens',statistics.stdev(MC_stack_sens))\n",
        "\n",
        "# Random forest classifier\n",
        "AC_rf_sens = []\n",
        "AM_rf_sens = []\n",
        "MC_rf_sens = []\n",
        "for i in range(len(final_rf_test_sens)):\n",
        "      AC_rf_sens.append(final_rf_test_sens[i][0]*100)\n",
        "      AM_rf_sens.append(final_rf_test_sens[i][1]*100)\n",
        "      MC_rf_sens.append(final_rf_test_sens[i][2]*100)\n",
        "# Compute mean random forest test sensitivity for each AD-CTL, AD-MCI and MCI-CTL binary classification problem                   \n",
        "Mean_AC_rf_sens = sum(AC_rf_sens)/len(AC_rf_sens)\n",
        "Mean_AM_rf_sens = sum(AM_rf_sens)/len(AM_rf_sens)\n",
        "Mean_MC_rf_sens = sum(MC_rf_sens)/len(MC_rf_sens)\n",
        "print('Mean_AC_rf_Tsens', Mean_AC_rf_sens)\n",
        "print('Mean_AM_rf_Tsens', Mean_AM_rf_sens)\n",
        "print('Mean_MC_rf_Tsens', Mean_MC_rf_sens)\n",
        "# Compute standard deviation of the random forest test sensitivity for each AD-CTL, AD-MCI and MCI-CTL binary classification problem    \n",
        "print('Std_AC_rf_Tsens',statistics.stdev(AC_rf_sens))\n",
        "print('Std_AC_rf_Tsens',statistics.stdev(AM_rf_sens))\n",
        "print('Std_AC_rf_Tsens',statistics.stdev(MC_rf_sens))\n",
        "\n",
        "# Bagging ensemble of classifiers\n",
        "AC_bagging_sens = []\n",
        "AM_bagging_sens = []\n",
        "MC_bagging_sens = []\n",
        "for i in range(len(final_bagging_test_sens)):\n",
        "      AC_bagging_sens.append(final_bagging_test_sens[i][0]*100)\n",
        "      AM_bagging_sens.append(final_bagging_test_sens[i][1]*100)\n",
        "      MC_bagging_sens.append(final_bagging_test_sens[i][2]*100)\n",
        "# Compute mean bagging ensemble test sensitivity for each AD-CTL, AD-MCI and MCI-CTL binary classification problem                   \n",
        "Mean_AC_bagging_sens = sum(AC_bagging_sens)/len(AC_bagging_sens)\n",
        "Mean_AM_bagging_sens = sum(AM_bagging_sens)/len(AM_bagging_sens)\n",
        "Mean_MC_bagging_sens = sum(MC_bagging_sens)/len(MC_bagging_sens)\n",
        "print('Mean_AC_bagging_sens', Mean_AC_bagging_sens)\n",
        "print('Mean_AM_bagging_sens', Mean_AM_bagging_sens)\n",
        "print('Mean_MC_bagging_sens', Mean_MC_bagging_sens)\n",
        "# Compute standard deviation of the bagging ensemble test sensitivity for each AD-CTL, AD-MCI and MCI-CTL binary classification problem    \n",
        "print('Std_AC_bagging_Tsens',statistics.stdev(AC_bagging_sens))\n",
        "print('Std_AC_bagging_Tsens',statistics.stdev(AM_bagging_sens))\n",
        "print('Std_AC_bagging_Tsens',statistics.stdev(MC_bagging_sens))\n",
        "\n",
        "# Naives bayes classifier\n",
        "AC_naivebayes_sens = []\n",
        "AM_naivebayes_sens = []\n",
        "MC_naivebayes_sens = []\n",
        "for i in range(len(final_naivebayes_test_sens)):\n",
        "      AC_naivebayes_sens.append(final_naivebayes_test_sens[i][0]*100)\n",
        "      AM_naivebayes_sens.append(final_naivebayes_test_sens[i][1]*100)\n",
        "      MC_naivebayes_sens.append(final_naivebayes_test_sens[i][2]*100)\n",
        "# Compute mean naive bayes test sensitivity for each AD-CTL, AD-MCI and MCI-CTL binary classification problem                   \n",
        "Mean_AC_naivebayes_sens = sum(AC_naivebayes_sens)/len(AC_naivebayes_sens)\n",
        "Mean_AM_naivebayes_sens = sum(AM_naivebayes_sens)/len(AM_naivebayes_sens)\n",
        "Mean_MC_naivebayes_sens = sum(MC_naivebayes_sens)/len(MC_naivebayes_sens)\n",
        "print('Mean_AC_naivebayes_sens', Mean_AC_naivebayes_sens)\n",
        "print('Mean_AM_naivebayes_sens', Mean_AM_naivebayes_sens)\n",
        "print('Mean_MC_naivebayes_sens', Mean_MC_naivebayes_sens)\n",
        "# Compute standard deviation of the naive bayes test sensitivity for each AD-CTL, AD-MCI and MCI-CTL binary classification problem    \n",
        "print('Std_AC_naivebayes_sens',statistics.stdev(AC_naivebayes_sens))\n",
        "print('Std_AM_naivebayes_sens',statistics.stdev(AM_naivebayes_sens))\n",
        "print('Std_MC_naivebayes_sens',statistics.stdev(MC_naivebayes_sens))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "q6dcdHiZdbNc",
        "colab_type": "text"
      },
      "source": [
        "**Compute mean and standard deviation of test specificity for each of these classifiers**\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TqYA1mzMdeYs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# LDA classifier\n",
        "AC_LDA_TSpec = []\n",
        "AM_LDA_TSpec = []\n",
        "MC_LDA_TSpec = []\n",
        "for i in range(len(final_LDA_test_spec)):\n",
        "      AC_LDA_TSpec.append(final_LDA_test_spec[i][0]*100)\n",
        "      AM_LDA_TSpec.append(final_LDA_test_spec[i][1]*100)\n",
        "      MC_LDA_TSpec.append(final_LDA_test_spec[i][2]*100)\n",
        "# Compute mean test specificity for each AD-CTL, AD-MCI and MCI-CTL binary classification problem                   \n",
        "Mean_AC_LDA_TSpec = sum(AC_LDA_TSpec)/len(AC_LDA_TSpec)\n",
        "Mean_AM_LDA_TSpec = sum(AM_LDA_TSpec)/len(AM_LDA_TSpec)\n",
        "Mean_MC_LDA_TSpec = sum(MC_LDA_TSpec)/len(MC_LDA_TSpec)\n",
        "print('Mean_AC_LDA_TSpec', Mean_AC_LDA_TSpec)\n",
        "print('Mean_AM_LDA_TSpec', Mean_AM_LDA_TSpec)\n",
        "print('Mean_MC_LDA_TSpec', Mean_MC_LDA_TSpec)   \n",
        "# Compute standard deviation of the test specificity for each AD-CTL, AD-MCI and MCI-CTL binary classification problem    \n",
        "print('Std_AC_LDA_TSpec', statistics.stdev(AC_LDA_TSpec))\n",
        "print('Std_AM_LDA_TSpec', statistics.stdev(AM_LDA_TSpec))\n",
        "print('Std_MC_LDA_TSpec', statistics.stdev(MC_LDA_TSpec))\n",
        "\n",
        "# LOgistic regression classifier\n",
        "AC_SGD_TSpec = []\n",
        "AM_SGD_TSpec = []\n",
        "MC_SGD_TSpec = []\n",
        "for i in range(len(final_SGD_test_spec)):\n",
        "      AC_SGD_TSpec.append(final_SGD_test_spec[i][0]*100)\n",
        "      AM_SGD_TSpec.append(final_SGD_test_spec[i][1]*100)\n",
        "      MC_SGD_TSpec.append(final_SGD_test_spec[i][2]*100)\n",
        "# Compute mean test specificity for each AD-CTL, AD-MCI and MCI-CTL binary classification problem                   \n",
        "Mean_AC_SGD_TSpec = sum(AC_SGD_TSpec)/len(AC_SGD_TSpec)\n",
        "Mean_AM_SGD_TSpec = sum(AM_SGD_TSpec)/len(AM_SGD_TSpec)\n",
        "Mean_MC_SGD_TSpec = sum(MC_SGD_TSpec)/len(MC_SGD_TSpec) \n",
        "print('Mean_AC_SGD_TSpec', Mean_AC_SGD_TSpec)\n",
        "print('Mean_AM_SGD_TSpec', Mean_AM_SGD_TSpec)\n",
        "print('Mean_MC_SGD_TSpec', Mean_MC_SGD_TSpec) \n",
        "# Compute standard deviation of the test specificity for each AD-CTL, AD-MCI and MCI-CTL binary classification problem    \n",
        "print('Std_AC_SGD_TSpec', statistics.stdev(AC_SGD_TSpec))\n",
        "print('Std_AM_SGD_TSpec', statistics.stdev(AM_SGD_TSpec))\n",
        "print('Std_MC_SGD_TSpec', statistics.stdev(MC_SGD_TSpec))\n",
        "\n",
        "# SVM classifier\n",
        "AC_SVM_TSpec = []\n",
        "AM_SVM_TSpec = []\n",
        "MC_SVM_TSpec = []\n",
        "for i in range(len(final_SVM_test_spec)):\n",
        "      AC_SVM_TSpec.append(final_SVM_test_spec[i][0]*100)\n",
        "      AM_SVM_TSpec.append(final_SVM_test_spec[i][1]*100)\n",
        "      MC_SVM_TSpec.append(final_SVM_test_spec[i][2]*100)\n",
        "# Compute mean test specificity for each AD-CTL, AD-MCI and MCI-CTL binary classification problem                      \n",
        "Mean_AC_SVM_TSpec = sum(AC_SVM_TSpec)/len(AC_SVM_TSpec)\n",
        "Mean_AM_SVM_TSpec = sum(AM_SVM_TSpec)/len(AM_SVM_TSpec)\n",
        "Mean_MC_SVM_TSpec = sum(MC_SVM_TSpec)/len(MC_SVM_TSpec) \n",
        "print('Mean_AC_SVM_TSpec', Mean_AC_SVM_TSpec)\n",
        "print('Mean_AM_SVM_TSpec', Mean_AM_SVM_TSpec)\n",
        "print('Mean_MC_SVM_TSpec', Mean_MC_SVM_TSpec)\n",
        "# Compute standard deviation of the test specificity for each AD-CTL, AD-MCI and MCI-CTL binary classification problem    \n",
        "print('Std_AC_SVM_TSpec', statistics.stdev(AC_SVM_TSpec))\n",
        "print('Std_AM_SVM_TSpec', statistics.stdev(AM_SVM_TSpec))\n",
        "print('Std_MC_SVM_TSpec', statistics.stdev(MC_SVM_TSpec))\n",
        "\n",
        "# Decision tree classifier\n",
        "AC_DT_TSpec = []\n",
        "AM_DT_TSpec = []\n",
        "MC_DT_TSpec = []\n",
        "for i in range(len(final_DT_test_spec)):\n",
        "      AC_DT_TSpec.append(final_DT_test_spec[i][0]*100)\n",
        "      AM_DT_TSpec.append(final_DT_test_spec[i][1]*100)\n",
        "      MC_DT_TSpec.append(final_DT_test_spec[i][2]*100)\n",
        "# Compute mean test specificity for each AD-CTL, AD-MCI and MCI-CTL binary classification problem                       \n",
        "Mean_AC_DT_TSpec = sum(AC_DT_TSpec)/len(AC_DT_TSpec)\n",
        "Mean_AM_DT_TSpec = sum(AM_DT_TSpec)/len(AM_DT_TSpec)\n",
        "Mean_MC_DT_TSpec = sum(MC_DT_TSpec)/len(MC_DT_TSpec)\n",
        "print('Mean_AC_DT_TSpec', Mean_AC_DT_TSpec)\n",
        "print('Mean_AM_DT_TSpec', Mean_AM_DT_TSpec)\n",
        "print('Mean_MC_DT_TSpec', Mean_MC_DT_TSpec)\n",
        "# Compute standard deviation of the test specificity for each AD-CTL, AD-MCI and MCI-CTL binary classification problem    \n",
        "print('Std_AC_DT_TSpec', statistics.stdev(AC_DT_TSpec))\n",
        "print('Std_AM_DT_TSpec', statistics.stdev(AM_DT_TSpec))\n",
        "print('Std_MC_DT_TSpec', statistics.stdev(MC_DT_TSpec))\n",
        "\n",
        "# KNN classifier\n",
        "AC_KNN_TSpec = []\n",
        "AM_KNN_TSpec = []\n",
        "MC_KNN_TSpec = []\n",
        "for i in range(len(final_KNN_test_spec)):\n",
        "      AC_KNN_TSpec.append(final_KNN_test_spec[i][0]*100)\n",
        "      AM_KNN_TSpec.append(final_KNN_test_spec[i][1]*100)\n",
        "      MC_KNN_TSpec.append(final_KNN_test_spec[i][2]*100)\n",
        "# Compute mean test specificity for each AD-CTL, AD-MCI and MCI-CTL binary classification problem                       \n",
        "Mean_AC_KNN_TSpec = sum(AC_KNN_TSpec)/len(AC_KNN_TSpec)\n",
        "Mean_AM_KNN_TSpec = sum(AM_KNN_TSpec)/len(AM_KNN_TSpec)\n",
        "Mean_MC_KNN_TSpec = sum(MC_KNN_TSpec)/len(MC_KNN_TSpec) \n",
        "print('Mean_AC_KNN_TSpec', Mean_AC_KNN_TSpec)\n",
        "print('Mean_AM_KNN_TSpec', Mean_AM_KNN_TSpec)\n",
        "print('Mean_MC_KNN_TSpec', Mean_MC_KNN_TSpec)\n",
        "# Compute standard deviation of the test specificity for each AD-CTL, AD-MCI and MCI-CTL binary classification problem    \n",
        "print('Mean_AC_KNN_TSpec', statistics.stdev(AC_KNN_TSpec))\n",
        "print('Mean_AM_KNN_TSpec', statistics.stdev(AM_KNN_TSpec))\n",
        "print('Mean_MC_KNN_TSpec', statistics.stdev(MC_KNN_TSpec))\n",
        "\n",
        "# Stacking ensemble of classifiers\n",
        "AC_stack_spec = []\n",
        "AM_stack_spec = []\n",
        "MC_stack_spec = []\n",
        "for i in range(len(final_stack_test_spec)):\n",
        "      AC_stack_spec.append(final_stack_test_spec[i][0]*100)\n",
        "      AM_stack_spec.append(final_stack_test_spec[i][1]*100)\n",
        "      MC_stack_spec.append(final_stack_test_spec[i][2]*100)\n",
        "# Compute mean test specificity for each AD-CTL, AD-MCI and MCI-CTL binary classification problem                        \n",
        "Mean_AC_stack_spec = sum(AC_stack_spec)/len(AC_stack_spec)\n",
        "Mean_AM_stack_spec = sum(AM_stack_spec)/len(AM_stack_spec)\n",
        "Mean_MC_stack_spec = sum(MC_stack_spec)/len(MC_stack_spec)\n",
        "print('Mean_AC_stack_spec', Mean_AC_stack_spec)\n",
        "print('Mean_AM_stack_spec', Mean_AM_stack_spec)\n",
        "print('Mean_MC_stack_spec', Mean_MC_stack_spec)\n",
        "# Compute standard deviation of the test specificity for each AD-CTL, AD-MCI and MCI-CTL binary classification problem    \n",
        "print('Std_AC_stack_Tspec',statistics.stdev(AC_stack_spec))\n",
        "print('Std_AM_stack_Tspec',statistics.stdev(AM_stack_spec))\n",
        "print('Std_MC_stack_Tspec',statistics.stdev(MC_stack_spec))\n",
        "\n",
        "# Random forest classifier\n",
        "AC_rf_spec = []\n",
        "AM_rf_spec = []\n",
        "MC_rf_spec = []\n",
        "for i in range(len(final_rf_test_spec)):\n",
        "      AC_rf_spec.append(final_rf_test_spec[i][0]*100)\n",
        "      AM_rf_spec.append(final_rf_test_spec[i][1]*100)\n",
        "      MC_rf_spec.append(final_rf_test_spec[i][2]*100)\n",
        "# Compute mean test specificity for each AD-CTL, AD-MCI and MCI-CTL binary classification problem                          \n",
        "Mean_AC_rf_spec = sum(AC_rf_spec)/len(AC_rf_spec)\n",
        "Mean_AM_rf_spec = sum(AM_rf_spec)/len(AM_rf_spec)\n",
        "Mean_MC_rf_spec = sum(MC_rf_spec)/len(MC_rf_spec)\n",
        "print('Mean_AC_rf_spec', Mean_AC_stack_spec)\n",
        "print('Mean_AM_rf_spec', Mean_AM_stack_spec)\n",
        "print('Mean_MC_rf_spec', Mean_MC_stack_spec)\n",
        "# Compute standard deviation of the test specificity for each AD-CTL, AD-MCI and MCI-CTL binary classification problem    \n",
        "print('Std_AC_rf_Tspec',statistics.stdev(AC_rf_spec))\n",
        "print('Std_AM_rf_Tspec',statistics.stdev(AM_rf_spec))\n",
        "print('Std_MC_rf_Tspec',statistics.stdev(MC_rf_spec))\n",
        "\n",
        "# Bagging ensemble of classifiers\n",
        "AC_bagging_spec = []\n",
        "AM_bagging_spec = []\n",
        "MC_bagging_spec = []\n",
        "for i in range(len(final_bagging_test_spec)):\n",
        "      AC_bagging_spec.append(final_bagging_test_spec[i][0]*100)\n",
        "      AM_bagging_spec.append(final_bagging_test_spec[i][1]*100)\n",
        "      MC_bagging_spec.append(final_bagging_test_spec[i][2]*100)\n",
        "# Compute mean test specificity for each AD-CTL, AD-MCI and MCI-CTL binary classification problem                          \n",
        "Mean_AC_bagging_spec = sum(AC_bagging_spec)/len(AC_bagging_spec)\n",
        "Mean_AM_bagging_spec = sum(AM_bagging_spec)/len(AM_bagging_spec)\n",
        "Mean_MC_bagging_spec = sum(MC_bagging_spec)/len(MC_bagging_spec)\n",
        "print('Mean_AC_bagging_spec', Mean_AC_bagging_spec)\n",
        "print('Mean_AM_bagging_spec', Mean_AM_bagging_spec)\n",
        "print('Mean_MC_bagging_spec', Mean_MC_bagging_spec)\n",
        "# Compute standard deviation of the test specificity for each AD-CTL, AD-MCI and MCI-CTL binary classification problem    \n",
        "print('Std_AC_bagging_spec',statistics.stdev(AC_bagging_spec))\n",
        "print('Std_AM_bagging_spec',statistics.stdev(AM_bagging_spec))\n",
        "print('Std_MC_bagging_spec',statistics.stdev(MC_bagging_spec))\n",
        "\n",
        "# Naive bayes classifier\n",
        "AC_naivebayes_spec = []\n",
        "AM_naivebayes_spec = []\n",
        "MC_naivebayes_spec = []\n",
        "for i in range(len(final_naivebayes_test_spec)):\n",
        "      AC_naivebayes_spec.append(final_naivebayes_test_spec[i][0]*100)\n",
        "      AM_naivebayes_spec.append(final_naivebayes_test_spec[i][1]*100)\n",
        "      MC_naivebayes_spec.append(final_naivebayes_test_spec[i][2]*100)\n",
        "# Compute mean test specificity for each AD-CTL, AD-MCI and MCI-CTL binary classification problem                          \n",
        "Mean_AC_naivebayes_spec = sum(AC_naivebayes_spec)/len(AC_naivebayes_spec)\n",
        "Mean_AM_naivebayes_spec = sum(AM_naivebayes_spec)/len(AM_naivebayes_spec)\n",
        "Mean_MC_naivebayes_spec = sum(MC_naivebayes_spec)/len(MC_naivebayes_spec)\n",
        "print('Mean_AC_naivebayes_spec', Mean_AC_naivebayes_spec)\n",
        "print('Mean_AM_naivebayes_spec', Mean_AM_naivebayes_spec)\n",
        "print('Mean_MC_naivebayes_spec', Mean_MC_naivebayes_spec)\n",
        "# Compute standard deviation of the test specificity for each AD-CTL, AD-MCI and MCI-CTL binary classification problem    \n",
        "print('Std_AC_naivebayes_spec',statistics.stdev(AC_naivebayes_spec))\n",
        "print('Std_AM_naivebayes_spec',statistics.stdev(AM_naivebayes_spec))\n",
        "print('Std_MC_naivebayes_spec',statistics.stdev(MC_naivebayes_spec))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KQbE7ZY-dgeP",
        "colab_type": "text"
      },
      "source": [
        "**Compute mean test F1 score for each of these classifiers**\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Fn46e3mPdi7V",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# LDA classifier\n",
        "AC_LDA_F = []\n",
        "AM_LDA_F = []\n",
        "MC_LDA_F = []\n",
        "for i in range(len(final_LDA_test_f1)):\n",
        "      AC_LDA_F.append(final_LDA_test_f1[i][0]*100)\n",
        "      AM_LDA_F.append(final_LDA_test_f1[i][1]*100)\n",
        "      MC_LDA_F.append(final_LDA_test_f1[i][2]*100)\n",
        "# Compute mean test F1 score for each AD-CTL, AD-MCI and MCI-CTL binary classification problem                               \n",
        "Mean_AC_LDA_F = sum(AC_LDA_F)/len(AC_LDA_F)\n",
        "Mean_AM_LDA_F = sum(AM_LDA_F)/len(AM_LDA_F)\n",
        "Mean_MC_LDA_F = sum(MC_LDA_F)/len(MC_LDA_F) \n",
        "print('Mean_AC_LDA_F', Mean_AC_LDA_F)\n",
        "print('Mean_AM_LDA_F', Mean_AM_LDA_F)\n",
        "print('Mean_MC_LDA_F', Mean_MC_LDA_F)\n",
        "\n",
        "# Logistic regression classifier\n",
        "AC_SGD_F = []\n",
        "AM_SGD_F = []\n",
        "MC_SGD_F = []\n",
        "for i in range(len(final_SGD_test_f1)):\n",
        "      AC_SGD_F.append(final_SGD_test_f1[i][0]*100)\n",
        "      AM_SGD_F.append(final_SGD_test_f1[i][1]*100)\n",
        "      MC_SGD_F.append(final_SGD_test_f1[i][2]*100)\n",
        "# Compute mean test F1 score for each AD-CTL, AD-MCI and MCI-CTL binary classification problem                                 \n",
        "Mean_AC_SGD_F = sum(AC_SGD_F)/len(AC_SGD_F)\n",
        "Mean_AM_SGD_F = sum(AM_SGD_F)/len(AM_SGD_F)\n",
        "Mean_MC_SGD_F = sum(MC_SGD_F)/len(MC_SGD_F) \n",
        "print('Mean_AC_SGD_F', Mean_AC_SGD_F)\n",
        "print('Mean_AM_SGD_F', Mean_AM_SGD_F)\n",
        "print('Mean_MC_SGD_F', Mean_MC_SGD_F)\n",
        "\n",
        "# SVM classifier\n",
        "AC_SVM_F = []\n",
        "AM_SVM_F = []\n",
        "MC_SVM_F = []\n",
        "for i in range(len(final_SVM_test_f1)):\n",
        "      AC_SVM_F.append(final_SVM_test_f1[i][0]*100)\n",
        "      AM_SVM_F.append(final_SVM_test_f1[i][1]*100)\n",
        "      MC_SVM_F.append(final_SVM_test_f1[i][2]*100)\n",
        "# Compute mean test F1 score for each AD-CTL, AD-MCI and MCI-CTL binary classification problem                                 \n",
        "Mean_AC_SVM_F = sum(AC_SVM_F)/len(AC_SVM_F)\n",
        "Mean_AM_SVM_F = sum(AM_SVM_F)/len(AM_SVM_F)\n",
        "Mean_MC_SVM_F = sum(MC_SVM_F)/len(MC_SVM_F) \n",
        "print('Mean_AC_SVM_F', Mean_AC_SVM_F)\n",
        "print('Mean_AM_SVM_F', Mean_AM_SVM_F)\n",
        "print('Mean_MC_SVM_F', Mean_MC_SVM_F)\n",
        "\n",
        "# Decision tree classifier\n",
        "AC_DT_F = []\n",
        "AM_DT_F = []\n",
        "MC_DT_F = []\n",
        "for i in range(len(final_DT_test_f1)):\n",
        "      AC_DT_F.append(final_DT_test_f1[i][0]*100)\n",
        "      AM_DT_F.append(final_DT_test_f1[i][1]*100)\n",
        "      MC_DT_F.append(final_DT_test_f1[i][2]*100)\n",
        "# Compute mean test F1 score for each AD-CTL, AD-MCI and MCI-CTL binary classification problem                                  \n",
        "Mean_AC_DT_F = sum(AC_DT_F)/len(AC_DT_F)\n",
        "Mean_AM_DT_F = sum(AM_DT_F)/len(AM_DT_F)\n",
        "Mean_MC_DT_F = sum(MC_DT_F)/len(MC_DT_F) \n",
        "print('Mean_AC_DT_F', Mean_AC_DT_F)\n",
        "print('Mean_AM_DT_F', Mean_AM_DT_F)\n",
        "print('Mean_MC_DT_F', Mean_MC_DT_F)\n",
        "\n",
        "# KNN classifier\n",
        "AC_KNN_F = []\n",
        "AM_KNN_F = []\n",
        "MC_KNN_F = []\n",
        "for i in range(len(final_KNN_test_f1)):\n",
        "      AC_KNN_F.append(final_KNN_test_f1[i][0]*100)\n",
        "      AM_KNN_F.append(final_KNN_test_f1[i][1]*100)\n",
        "      MC_KNN_F.append(final_KNN_test_f1[i][2]*100)\n",
        "# Compute mean test F1 score for each AD-CTL, AD-MCI and MCI-CTL binary classification problem                               \n",
        "Mean_AC_KNN_F = sum(AC_KNN_F)/len(AC_KNN_F)\n",
        "Mean_AM_KNN_F = sum(AM_KNN_F)/len(AM_KNN_F)\n",
        "Mean_MC_KNN_F = sum(MC_KNN_F)/len(MC_KNN_F) \n",
        "print('Mean_AC_KNN_F', Mean_AC_KNN_F)\n",
        "print('Mean_AM_KNN_F', Mean_AM_KNN_F)\n",
        "print('Mean_MC_KNN_F', Mean_MC_KNN_F)\n",
        "\n",
        "# Stacking ensemble of classifiers\n",
        "AC_stack_f1 = []\n",
        "AM_stack_f1 = []\n",
        "MC_stack_f1 = []\n",
        "for i in range(len(final_stack_test_f1)):\n",
        "      AC_stack_f1.append(final_stack_test_f1[i][0]*100)\n",
        "      AM_stack_f1.append(final_stack_test_f1[i][1]*100)\n",
        "      MC_stack_f1.append(final_stack_test_f1[i][2]*100)\n",
        "# Compute mean test F1 score for each AD-CTL, AD-MCI and MCI-CTL binary classification problem                                  \n",
        "Mean_AC_stack_f1 = sum(AC_stack_f1)/len(AC_stack_f1)\n",
        "Mean_AM_stack_f1 = sum(AM_stack_f1)/len(AM_stack_f1)\n",
        "Mean_MC_stack_f1 = sum(MC_stack_f1)/len(MC_stack_f1)\n",
        "print('Mean_AC_stack_f1', Mean_AC_stack_f1)\n",
        "print('Mean_AM_stack_f1', Mean_AM_stack_f1)\n",
        "print('Mean_MC_stack_f1', Mean_MC_stack_f1)\n",
        "\n",
        "# Random forest classifier\n",
        "AC_rf_f1 = []\n",
        "AM_rf_f1 = []\n",
        "MC_rf_f1 = []\n",
        "for i in range(len(final_rf_test_f1)):\n",
        "      AC_rf_f1.append(final_rf_test_f1[i][0]*100)\n",
        "      AM_rf_f1.append(final_rf_test_f1[i][1]*100)\n",
        "      MC_rf_f1.append(final_rf_test_f1[i][2]*100)\n",
        "# Compute mean test F1 score for each AD-CTL, AD-MCI and MCI-CTL binary classification problem                                     \n",
        "Mean_AC_rf_f1 = sum(AC_rf_f1)/len(AC_rf_f1)\n",
        "Mean_AM_rf_f1 = sum(AM_rf_f1)/len(AM_rf_f1)\n",
        "Mean_MC_rf_f1 = sum(MC_rf_f1)/len(MC_rf_f1)\n",
        "print('Mean_AC_rf_f1', Mean_AC_rf_f1)\n",
        "print('Mean_AM_rf_f1', Mean_AM_rf_f1)\n",
        "print('Mean_MC_rf_f1', Mean_MC_rf_f1)\n",
        "\n",
        "# Bagging ensemble of classifiers\n",
        "AC_bagging_f1 = []\n",
        "AM_bagging_f1 = []\n",
        "MC_bagging_f1 = []\n",
        "for i in range(len(final_bagging_test_f1)):\n",
        "      AC_bagging_f1.append(final_bagging_test_f1[i][0]*100)\n",
        "      AM_bagging_f1.append(final_bagging_test_f1[i][1]*100)\n",
        "      MC_bagging_f1.append(final_bagging_test_f1[i][2]*100)\n",
        "# Compute mean test F1 score for each AD-CTL, AD-MCI and MCI-CTL binary classification problem                                        \n",
        "Mean_AC_bagging_f1 = sum(AC_bagging_f1)/len(AC_bagging_f1)\n",
        "Mean_AM_bagging_f1 = sum(AM_bagging_f1)/len(AM_bagging_f1)\n",
        "Mean_MC_bagging_f1 = sum(MC_bagging_f1)/len(MC_bagging_f1)\n",
        "print('Mean_AC_bagging_f1', Mean_AC_bagging_f1)\n",
        "print('Mean_AM_bagging_f1', Mean_AM_bagging_f1)\n",
        "print('Mean_MC_bagging_f1', Mean_MC_bagging_f1)\n",
        "\n",
        "# Naive bayes classifier\n",
        "AC_naivebayes_f1 = []\n",
        "AM_naivebayes_f1 = []\n",
        "MC_naivebayes_f1 = []\n",
        "for i in range(len(final_naivebayes_test_f1)):\n",
        "      AC_naivebayes_f1.append(final_naivebayes_test_f1[i][0]*100)\n",
        "      AM_naivebayes_f1.append(final_naivebayes_test_f1[i][1]*100)\n",
        "      MC_naivebayes_f1.append(final_naivebayes_test_f1[i][2]*100)\n",
        "# Compute mean test F1 score for each AD-CTL, AD-MCI and MCI-CTL binary classification problem                                           \n",
        "Mean_AC_naivebayes_f1 = sum(AC_naivebayes_f1)/len(AC_naivebayes_f1)\n",
        "Mean_AM_naivebayes_f1 = sum(AM_naivebayes_f1)/len(AM_naivebayes_f1)\n",
        "Mean_MC_naivebayes_f1 = sum(MC_naivebayes_f1)/len(MC_naivebayes_f1)\n",
        "print('Mean_AC_naivebayes_f1', Mean_AC_naivebayes_f1)\n",
        "print('Mean_AM_naivebayes_f1', Mean_AM_naivebayes_f1)\n",
        "print('Mean_MC_naivebayes_f1', Mean_MC_naivebayes_f1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "x6s25ccIdkyU",
        "colab_type": "text"
      },
      "source": [
        "**Compute mean test error for each of these classifiers**\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HgkWK2MqdnSd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# LDA classifier\n",
        "AC_LDA_E = []\n",
        "AM_LDA_E = []\n",
        "MC_LDA_E = []\n",
        "for i in range(len(final_LDA_test_err)):\n",
        "      AC_LDA_E.append(final_LDA_test_err[i][0]*100)\n",
        "      AM_LDA_E.append(final_LDA_test_err[i][1]*100)\n",
        "      MC_LDA_E.append(final_LDA_test_err[i][2]*100)\n",
        "# Compute mean test error for each AD-CTL, AD-MCI and MCI-CTL binary classification problem                                                \n",
        "Mean_AC_LDA_E = sum(AC_LDA_E)/len(AC_LDA_E)\n",
        "Mean_AM_LDA_E = sum(AM_LDA_E)/len(AM_LDA_E)\n",
        "Mean_MC_LDA_E = sum(MC_LDA_E)/len(MC_LDA_E)  \n",
        "print('Mean_AC_LDA_E', Mean_AC_LDA_E)\n",
        "print('Mean_AM_LDA_E', Mean_AM_LDA_E)\n",
        "print('Mean_MC_LDA_E', Mean_MC_LDA_E)\n",
        "\n",
        "# Logistic regression classifer\n",
        "AC_SGD_E = []\n",
        "AM_SGD_E = []\n",
        "MC_SGD_E = []\n",
        "for i in range(len(final_SGD_test_err)):\n",
        "      AC_SGD_E.append(final_SGD_test_err[i][0]*100)\n",
        "      AM_SGD_E.append(final_SGD_test_err[i][1]*100)\n",
        "      MC_SGD_E.append(final_SGD_test_err[i][2]*100)\n",
        "# Compute mean test error for each AD-CTL, AD-MCI and MCI-CTL binary classification problem                                                    \n",
        "Mean_AC_SGD_E = sum(AC_SGD_E)/len(AC_SGD_E)\n",
        "Mean_AM_SGD_E = sum(AM_SGD_E)/len(AM_SGD_E)\n",
        "Mean_MC_SGD_E = sum(MC_SGD_E)/len(MC_SGD_E) \n",
        "print('Mean_AC_SGD_E', Mean_AC_SGD_E)\n",
        "print('Mean_AM_SGD_E', Mean_AM_SGD_E)\n",
        "print('Mean_MC_SGD_E', Mean_MC_SGD_E)\n",
        "\n",
        "# SVM classifier\n",
        "AC_SVM_E = []\n",
        "AM_SVM_E = []\n",
        "MC_SVM_E = []\n",
        "for i in range(len(final_SVM_test_err)):\n",
        "      AC_SVM_E.append(final_SVM_test_err[i][0]*100)\n",
        "      AM_SVM_E.append(final_SVM_test_err[i][1]*100)\n",
        "      MC_SVM_E.append(final_SVM_test_err[i][2]*100)\n",
        "# Compute mean test error for each AD-CTL, AD-MCI and MCI-CTL binary classification problem                                                    \n",
        "Mean_AC_SVM_E = sum(AC_SVM_E)/len(AC_SVM_E)\n",
        "Mean_AM_SVM_E = sum(AM_SVM_E)/len(AM_SVM_E)\n",
        "Mean_MC_SVM_E = sum(MC_SVM_E)/len(MC_SVM_E) \n",
        "print('Mean_AC_SVM_E', Mean_AC_SVM_E)\n",
        "print('Mean_AM_SVM_E', Mean_AM_SVM_E)\n",
        "print('Mean_MC_SVM_E', Mean_MC_SVM_E)\n",
        "\n",
        "# Decision tree classifier\n",
        "AC_DT_E = []\n",
        "AM_DT_E = []\n",
        "MC_DT_E = []\n",
        "for i in range(len(final_DT_test_err)):\n",
        "      AC_DT_E.append(final_DT_test_err[i][0]*100)\n",
        "      AM_DT_E.append(final_DT_test_err[i][1]*100)\n",
        "      MC_DT_E.append(final_DT_test_err[i][2]*100)\n",
        "# Compute mean test error for each AD-CTL, AD-MCI and MCI-CTL binary classification problem                                                    \n",
        "Mean_AC_DT_E = sum(AC_DT_E)/len(AC_DT_E)\n",
        "Mean_AM_DT_E = sum(AM_DT_E)/len(AM_DT_E)\n",
        "Mean_MC_DT_E = sum(MC_DT_E)/len(MC_DT_E) \n",
        "print('Mean_AC_DT_E', Mean_AC_DT_E)\n",
        "print('Mean_AM_DT_E', Mean_AM_DT_E)\n",
        "print('Mean_MC_DT_E', Mean_MC_DT_E)\n",
        "\n",
        "# KNN classifier\n",
        "AC_KNN_E = []\n",
        "AM_KNN_E = []\n",
        "MC_KNN_E = []\n",
        "for i in range(len(final_KNN_test_err)):\n",
        "      AC_KNN_E.append(final_KNN_test_err[i][0]*100)\n",
        "      AM_KNN_E.append(final_KNN_test_err[i][1]*100)\n",
        "      MC_KNN_E.append(final_KNN_test_err[i][2]*100)\n",
        "# Compute mean test error for each AD-CTL, AD-MCI and MCI-CTL binary classification problem                                                    \n",
        "Mean_AC_KNN_E = sum(AC_KNN_E)/len(AC_KNN_E)\n",
        "Mean_AM_KNN_E = sum(AM_KNN_E)/len(AM_KNN_E)\n",
        "Mean_MC_KNN_E = sum(MC_KNN_E)/len(MC_KNN_E) \n",
        "print('Mean_AC_KNN_E', Mean_AC_KNN_E)\n",
        "print('Mean_AM_KNN_E', Mean_AM_KNN_E)\n",
        "print('Mean_MC_KNN_E', Mean_MC_KNN_E)\n",
        "\n",
        "# Stacking ensemble of classifiers\n",
        "AC_stack_err = []\n",
        "AM_stack_err = []\n",
        "MC_stack_err = []\n",
        "for i in range(len(final_stack_test_err)):\n",
        "      AC_stack_err.append(final_stack_test_err[i][0]*100)\n",
        "      AM_stack_err.append(final_stack_test_err[i][1]*100)\n",
        "      MC_stack_err.append(final_stack_test_err[i][2]*100)\n",
        "# Compute mean test error for each AD-CTL, AD-MCI and MCI-CTL binary classification problem                                                       \n",
        "Mean_AC_stack_err = sum(AC_stack_err)/len(AC_stack_err)\n",
        "Mean_AM_stack_err = sum(AM_stack_err)/len(AM_stack_err)\n",
        "Mean_MC_stack_err = sum(MC_stack_err)/len(MC_stack_err)\n",
        "print('Mean_AC_stack_err', Mean_AC_stack_err)\n",
        "print('Mean_AM_stack_err', Mean_AM_stack_err)\n",
        "print('Mean_MC_stack_err', Mean_MC_stack_err)\n",
        "\n",
        "# Random forest\n",
        "AC_rf_err = []\n",
        "AM_rf_err = []\n",
        "MC_rf_err = []\n",
        "for i in range(len(final_rf_test_err)):\n",
        "      AC_rf_err.append(final_rf_test_err[i][0]*100)\n",
        "      AM_rf_err.append(final_rf_test_err[i][1]*100)\n",
        "      MC_rf_err.append(final_rf_test_err[i][2]*100)\n",
        "# Compute mean test error for each AD-CTL, AD-MCI and MCI-CTL binary classification problem                                                          \n",
        "Mean_AC_rf_err = sum(AC_rf_err)/len(AC_rf_err)\n",
        "Mean_AM_rf_err = sum(AM_rf_err)/len(AM_rf_err)\n",
        "Mean_MC_rf_err = sum(MC_rf_err)/len(MC_rf_err)\n",
        "print('Mean_AC_rf_err', Mean_AC_rf_err)\n",
        "print('Mean_AM_rf_err', Mean_AM_rf_err)\n",
        "print('Mean_MC_rf_err', Mean_MC_rf_err)\n",
        "\n",
        "# Bagging ensemble of classifiers\n",
        "AC_bagging_err = []\n",
        "AM_bagging_err = []\n",
        "MC_bagging_err = []\n",
        "for i in range(len(final_bagging_test_err)):\n",
        "      AC_bagging_err.append(final_bagging_test_err[i][0]*100)\n",
        "      AM_bagging_err.append(final_bagging_test_err[i][1]*100)\n",
        "      MC_bagging_err.append(final_bagging_test_err[i][2]*100)\n",
        "# Compute mean test error for each AD-CTL, AD-MCI and MCI-CTL binary classification problem                                                             \n",
        "Mean_AC_bagging_err = sum(AC_bagging_err)/len(AC_bagging_err)\n",
        "Mean_AM_bagging_err = sum(AM_bagging_err)/len(AM_bagging_err)\n",
        "Mean_MC_bagging_err = sum(MC_bagging_err)/len(MC_bagging_err)\n",
        "print('Mean_AC_bagging_err', Mean_AC_bagging_err)\n",
        "print('Mean_AM_bagging_err', Mean_AM_bagging_err)\n",
        "print('Mean_MC_bagging_err', Mean_MC_bagging_err)\n",
        "\n",
        "# Naive bayes classifier\n",
        "AC_naivebayes_err = []\n",
        "AM_naivebayes_err = []\n",
        "MC_naivebayes_err = []\n",
        "for i in range(len(final_naivebayes_test_err)):\n",
        "      AC_naivebayes_err.append(final_naivebayes_test_err[i][0]*100)\n",
        "      AM_naivebayes_err.append(final_naivebayes_test_err[i][1]*100)\n",
        "      MC_naivebayes_err.append(final_naivebayes_test_err[i][2]*100)\n",
        "# Compute mean test error for each AD-CTL, AD-MCI and MCI-CTL binary classification problem                                                                \n",
        "Mean_AC_naivebayes_err = sum(AC_naivebayes_err)/len(AC_naivebayes_err)\n",
        "Mean_AM_naivebayes_err = sum(AM_naivebayes_err)/len(AM_naivebayes_err)\n",
        "Mean_MC_naivebayes_err = sum(MC_naivebayes_err)/len(MC_naivebayes_err)\n",
        "print('Mean_AC_naivebayes_err', Mean_AC_naivebayes_err)\n",
        "print('Mean_AM_naivebayes_err', Mean_AM_naivebayes_err)\n",
        "print('Mean_MC_naivebayes_err', Mean_MC_naivebayes_err)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mxBcyOQHdpLz",
        "colab_type": "text"
      },
      "source": [
        "**Compute the confusion matrices**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3vnK96mEdsqJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import seaborn as sns\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "# Uncomment the required classifier to compute the confusion matrix\n",
        "TP = sum(TP_lda)\n",
        "FP = sum(FP_lda)\n",
        "TN = sum(TN_lda)\n",
        "FN = sum(FN_lda)\n",
        "# TP = sum(TP_sgd)\n",
        "# FP = sum(FP_sgd)\n",
        "# TN = sum(TN_sgd)\n",
        "# FN = sum(FN_sgd)\n",
        "# TP = sum(TP_svm)\n",
        "# FP = sum(FP_svm)\n",
        "# TN = sum(TN_svm)\n",
        "# FN = sum(FN_svm)\n",
        "# TP = sum(TP_dt)\n",
        "# FP = sum(FP_dt)\n",
        "# TN = sum(TN_dt)\n",
        "# FN = sum(FN_dt)\n",
        "# TP = sum(TP_knn)\n",
        "# FP = sum(FP_knn)\n",
        "# TN = sum(TN_knn)\n",
        "# FN = sum(FN_knn)\n",
        "# TP = sum(TP_stack)\n",
        "# FP = sum(FP_stack)\n",
        "# TN = sum(TN_stack)\n",
        "# FN = sum(FN_stack)\n",
        "# TP = sum(TP_rf)\n",
        "# FP = sum(FP_rf)\n",
        "# TN = sum(TN_rf)\n",
        "# FN = sum(FN_rf)\n",
        "# TP = sum(TP_bag)\n",
        "# FP = sum(FP_bag)\n",
        "# TN = sum(TN_bag)\n",
        "# FN = sum(FN_bag)\n",
        "# TP = sum(TP_nb)\n",
        "# FP = sum(FP_nb)\n",
        "# TN = sum(TN_nb)\n",
        "# FN = sum(FN_nb)\n",
        "\n",
        "cm = [[TP,FP],\n",
        "         [FN,TN]]\n",
        "cm_df = pd.DataFrame(cm)\n",
        "\n",
        "# Plot confusion matrix\n",
        "sns.heatmap(cm_df, annot=True, cmap=\"Blues\", fmt='g', annot_kws={\"fontsize\":16})\n",
        "plt.ylabel('Predicted values')\n",
        "plt.xlabel('Actual values')\n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "c1CAc76Hdufr",
        "colab_type": "text"
      },
      "source": [
        "**Visualize the AD-CTL data in a 3D plane**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "znThrx2fdxdM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "X = X_train\n",
        "Y = y_train_1\n",
        "\n",
        "ad = []\n",
        "ctl = []\n",
        "x = []\n",
        "y = []\n",
        "z = []\n",
        "\n",
        "for i in range(len(X)):\n",
        "    if Y[i] == 1:\n",
        "        ad.append(X[i,339])       \n",
        "    elif Y[i] == 3:\n",
        "        ctl.append(X[i,534]) \n",
        "\n",
        "x = X[:,332]  \n",
        "y = X[:,104]\n",
        "z = X[:,3]      \n",
        "\n",
        "ad_test = []\n",
        "ctl_test = []\n",
        "# Change ypred_test according to the predicted values of the respective classifier for the AD-CTL data\n",
        "for i in range(len(X_test)):\n",
        "    if ypred_test[i] == 1:\n",
        "        ad_test.append(X_test[i,339])\n",
        "    elif ypred_test[i] == 3:    \n",
        "        ctl_test.append(X_test[i,534])    "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SmQnXTqTdzEm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from mpl_toolkits import mplot3d\n",
        "\n",
        "fig = plt.figure(figsize=(8,6))\n",
        "ax = fig.add_subplot(111, projection='3d')\n",
        "\n",
        "xs = ad[0:15]\n",
        "zs = ctl[0:15]\n",
        "\n",
        "xs_test = ad_test[0:3]\n",
        "zs_test = ctl_test[0:3]\n",
        "\n",
        "xt_test = x[0:3]\n",
        "yt_test = y[0:3]\n",
        "zt_test = z[0:3]\n",
        "\n",
        "xt = x[0:15]\n",
        "yt = y[0:15]\n",
        "zt = z[0:15]\n",
        "\n",
        "ax.scatter(zs, xt, zt, c ='y', marker='*', label='CTL')\n",
        "ax.scatter(xs, yt, zt, c ='r', marker='o', label='AD')\n",
        "ax.scatter(zs_test, xt_test, zt_test, c ='y', marker='D', s=120, label='Unseen CTL')\n",
        "ax.scatter(xs_test, yt_test, zt_test,c ='r', marker='P', s=120, label='Unseen AD')\n",
        "\n",
        "ax.set_xlabel('X-component')\n",
        "ax.set_ylabel('Log of Jacobian')\n",
        "ax.set_zlabel('Symmetry of Jacobian')\n",
        "ax.legend()\n",
        "\n",
        "plt.rcParams[\"axes.linewidth\"]  = 0.55\n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7YX-3Mx5d0wf",
        "colab_type": "text"
      },
      "source": [
        "**Visualize the AD-MCI data in a 3D plane**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "08gYppzTd3kR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "X = X_train\n",
        "Y = y_train_1\n",
        "\n",
        "ad = []\n",
        "mci = []\n",
        "x = []\n",
        "y = []\n",
        "z = []\n",
        "\n",
        "for i in range(len(X)):\n",
        "    if Y[i] == 1:\n",
        "        ad.append(X[i,200])       \n",
        "    elif Y[i] == 2:\n",
        "        mci.append(X[i,766]) \n",
        "\n",
        "x = X[:,164]  \n",
        "y = X[:,173]\n",
        "z = X[:,18]      \n",
        "\n",
        "ad_test = []\n",
        "mci_test = []\n",
        "# Change ypred_test according to the predicted values of the respective classifier\n",
        "for i in range(len(X_test)):\n",
        "    if ypred_test[i] == 1:\n",
        "        ad_test.append(X_test[i,200])\n",
        "    elif ypred_test[i] == 2:    \n",
        "        mci_test.append(X_test[i,766])    "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iA55DTrrd5DV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from mpl_toolkits import mplot3d\n",
        "\n",
        "fig = plt.figure(figsize=(8,6))\n",
        "ax = fig.add_subplot(111, projection='3d')\n",
        "\n",
        "xs = ad[0:15]\n",
        "ys = mci[0:15]\n",
        "\n",
        "xt = x[0:15]\n",
        "yt = y[0:15]\n",
        "zt = z[0:15]\n",
        "\n",
        "xs_test = ad_test[0:2]\n",
        "ys_test = mci_test[0:2]\n",
        "\n",
        "xt_test = x[0:2]\n",
        "yt_test = y[0:2]\n",
        "zt_test = z[0:2]\n",
        "\n",
        "ax.scatter(ys, xt, zt, c='r', marker='o', label='AD')\n",
        "ax.scatter(xs, yt, zt, c='b', marker='^', label='MCI')\n",
        "ax.scatter(ys_test, yt_test, zt_test, c ='b', marker='h', s=120, label='Unseen MCI')\n",
        "ax.scatter(xs_test, xt_test, zt_test, c ='r', marker='P', s=120, label='Unseen AD')\n",
        "\n",
        "ax.set_xlabel('X-component')\n",
        "ax.set_ylabel('Log of Jacobian')\n",
        "ax.set_zlabel('Symmetry of Jacobian')\n",
        "ax.legend()\n",
        "\n",
        "plt.rcParams[\"axes.linewidth\"]  = 0.55\n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QWNZiMuAd7O4",
        "colab_type": "text"
      },
      "source": [
        "**Visualize the MCI-CTL data in a 3D plane**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zFDeAqj6d-u6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "X = X_train\n",
        "Y = y_train_1\n",
        "\n",
        "mci = []\n",
        "ctl = []\n",
        "x = []\n",
        "y = []\n",
        "z = []\n",
        "\n",
        "for i in range(len(X)):\n",
        "    if Y[i] == 2:\n",
        "        mci.append(X[i,50])       \n",
        "    elif Y[i] == 3:\n",
        "        ctl.append(X[i,549]) \n",
        "\n",
        "x = X[:,2]  \n",
        "y = X[:,62]\n",
        "z = X[:,115]      \n",
        "\n",
        "mci_test = []\n",
        "ctl_test = []\n",
        "# Change ypred_test according to the predicted values of the respective classifier\n",
        "for i in range(len(X_test)):\n",
        "    if ypred_test[i] == 2:\n",
        "        ad_test.append(X_test[i,50])\n",
        "    elif ypred_test[i] == 3:    \n",
        "        mci_test.append(X_test[i,549])    "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JQ-uDos-eAcg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from mpl_toolkits import mplot3d\n",
        "\n",
        "fig = plt.figure(figsize=(8,6))\n",
        "ax = fig.add_subplot(111, projection='3d')\n",
        "\n",
        "ys = mci[0:15]\n",
        "zs = ctl[0:15]\n",
        "\n",
        "xt = x[0:15]\n",
        "yt = y[0:15]\n",
        "zt = z[0:15]\n",
        "\n",
        "ys_test = mci_test[0:3]\n",
        "zs_test = ctl_test[0:3]\n",
        "\n",
        "xt_test = x[0:3]\n",
        "yt_test = y[0:3]\n",
        "zt_test = z[0:3]\n",
        "\n",
        "ax.scatter(ys, yt, xt, c='y', marker='*', label='CTL')\n",
        "ax.scatter(zs, yt, xt, c='b', marker='^', label='MCI')\n",
        "ax.scatter(ys_test, yt_test, zt_test, c ='b', marker='h', s=120, label='Unseen MCI')\n",
        "ax.scatter(zs_test, yt_test, zt_test, c ='y', marker='D', s=120, label='Unseen CTL')\n",
        "\n",
        "ax.set_xlabel('X-component')\n",
        "ax.set_ylabel('Log of Jacobian')\n",
        "ax.set_zlabel('Symmetry of Jacobian')\n",
        "ax.legend()\n",
        "\n",
        "plt.rcParams[\"axes.linewidth\"]  = 0.55\n",
        "\n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TvtKfhRDSNwg",
        "colab_type": "text"
      },
      "source": [
        "**Visualize the AD, MCI and CTL data in a 3D plane**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rWHKRcq0SVgs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from mpl_toolkits import mplot3d\n",
        "\n",
        "fig = plt.figure(figsize=(8,6))\n",
        "ax = fig.add_subplot(111, projection='3d')\n",
        "\n",
        "xs = ad[0:15]\n",
        "ys = mci[0:15]\n",
        "zs = ctl[0:15]\n",
        "\n",
        "xt = x[0:15]\n",
        "yt = y[0:15]\n",
        "zt = z[0:15]\n",
        "\n",
        "ax.scatter(xs, yt, zt, c='r', marker='o', label='AD')\n",
        "ax.scatter(ys, xt, zt, c='b', marker='^', label='MCI')\n",
        "ax.scatter(zs, xt, yt, c ='y', marker='*', label='CTL')\n",
        "\n",
        "ax.set_xlabel('X-component')\n",
        "ax.set_ylabel('Log of Jacobian')\n",
        "ax.set_zlabel('Symmetry of Jacobian')\n",
        "ax.legend()\n",
        "\n",
        "plt.rcParams[\"axes.linewidth\"]  = 0.55\n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GmGDWfj3eChL",
        "colab_type": "text"
      },
      "source": [
        "**Plot box plots for test accuracies, sensitivity and specificity : AD-CTL data**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MyMs7e4IeFeN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from matplotlib import pyplot\n",
        "import seaborn as sns\n",
        "import pandas as pd\n",
        "\n",
        "data_to_plot = [AC_LDA_TA, AC_SGD_TA, AC_SVM_TA, AC_DT_TA, AC_KNN_TA, AC_stack_TA, AC_rf_TA, AC_bagging_TA, AC_naivebayes_TA]\n",
        "labels = ['LDA', 'Logistic regression', 'SVM', 'DT', 'KNN', 'Stacking ensemble', 'Random forest', 'Bagging ensemble', 'Naive bayes']\n",
        "\n",
        "\n",
        "fig = plt.figure(figsize=(16,6))\n",
        "ax = fig.add_subplot(111)\n",
        "medianprops = dict(linestyle='-', linewidth=2.5, color='firebrick')\n",
        "meanprops = dict(marker='h', markerfacecolor='b')\n",
        "\n",
        "bplot = pyplot.boxplot(data_to_plot, labels=labels, patch_artist=True, medianprops=medianprops, showmeans=True, meanprops=meanprops)\n",
        "\n",
        "ax.set_title('AD - CTL classification')\n",
        "ax.set_xlabel('Classifiers')\n",
        "ax.set_ylabel('Test accuracy')\n",
        "ax.yaxis.grid(True)\n",
        "cm = plt.cm.get_cmap('Pastel1')\n",
        "colors = [cm(val/len(data_to_plot)) for val in range(len(data_to_plot))]\n",
        "\n",
        "for patch, color in zip(bplot['boxes'], colors):\n",
        "    patch.set_facecolor(color)\n",
        "\n",
        "pyplot.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JqyVJF9deHlq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "\n",
        "data_a = [AC_LDA_TSens, AC_SGD_TSens, AC_SVM_TSens, AC_DT_TSens, AC_KNN_TSens, AC_stack_sens, AC_rf_sens, AC_bagging_sens, AC_naivebayes_sens]\n",
        "data_b = [AC_LDA_TSpec, AC_SGD_TSpec, AC_SVM_TSpec, AC_DT_TSpec, AC_KNN_TSpec, AC_stack_spec, AC_rf_spec, AC_bagging_spec, AC_naivebayes_spec]\n",
        "ticks = ['LDA', 'Logistic regression', 'SVM', 'DT', 'KNN', 'Stacking ensemble', 'Random forest', 'Bagging ensemble', 'Naive bayes']\n",
        "\n",
        "def set_box_color(bp, color):\n",
        "    plt.setp(bp['boxes'], color=color)\n",
        "    plt.setp(bp['whiskers'], color=color)\n",
        "    plt.setp(bp['caps'], color=color)\n",
        "    plt.setp(bp['medians'], color=color)\n",
        "\n",
        "plt.figure(figsize=(16,6))\n",
        "\n",
        "ax = fig.add_subplot(111)\n",
        "medianprops = dict(linestyle='-', linewidth=2.5, color='firebrick')\n",
        "meanprops = dict(marker='h', markerfacecolor='b')\n",
        "whiskerprops = dict(linestyle='-', linewidth=1)\n",
        "capprops = dict(linestyle='-', linewidth=1)\n",
        "\n",
        "bpl = plt.boxplot(data_a, positions=np.array(range(len(data_a)))*2.0-0.4, sym='', widths=0.6, patch_artist=True, medianprops=medianprops, showmeans=True, meanprops=meanprops, whiskerprops=whiskerprops, capprops=capprops)\n",
        "bpr = plt.boxplot(data_b, positions=np.array(range(len(data_b)))*2.0+0.4, sym='', widths=0.6, patch_artist=True, medianprops=medianprops, showmeans=True, meanprops=meanprops, whiskerprops=whiskerprops, capprops=capprops)\n",
        "set_box_color(bpl, 'firebrick')\n",
        "set_box_color(bpr, 'black')\n",
        "\n",
        "cm = plt.cm.get_cmap('Pastel1')\n",
        "colors = [cm(val/len(data_to_plot)) for val in range(len(data_to_plot))]\n",
        "for patch, color in zip(bpl['boxes'], colors):\n",
        "    patch.set_facecolor(color)\n",
        "for patch, color in zip(bpr['boxes'], colors):\n",
        "    patch.set_facecolor(color)\n",
        "\n",
        "plt.plot([], c='firebrick', label='Test Sensitivity')\n",
        "plt.plot([], c='black', label='Test Specificity')\n",
        "plt.legend()\n",
        "ax = plt.gca()\n",
        "ax.grid(True)\n",
        "plt.xticks(range(0, len(ticks) * 2, 2), ticks)\n",
        "ax.set_title('AD - CTL classification')\n",
        "ax.set_xlabel('Classifiers')\n",
        "plt.tight_layout()\n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XC_XPT5feJhp",
        "colab_type": "text"
      },
      "source": [
        "**Plot box plots for test accuracies, sensitivity and specificity : AD-MCI data**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DdsBkKBjePLX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from matplotlib import pyplot\n",
        "import seaborn as sns\n",
        "import pandas as pd\n",
        "\n",
        "data_to_plot = [AM_LDA_TA, AM_SGD_TA, AM_SVM_TA, AM_DT_TA, AM_KNN_TA, AM_stack_TA, AM_rf_TA, AM_bagging_TA, AM_naivebayes_TA]\n",
        "labels = ['LDA', 'Logistic regression', 'SVM', 'DT', 'KNN', 'Stacking ensemble', 'Random forest', 'Bagging ensemble', 'Naive bayes']\n",
        "\n",
        "\n",
        "fig = plt.figure(figsize=(16,6))\n",
        "ax = fig.add_subplot(111)\n",
        "medianprops = dict(linestyle='-', linewidth=2.5, color='firebrick')\n",
        "meanprops = dict(marker='h', markerfacecolor='b')\n",
        "\n",
        "bplot = pyplot.boxplot(data_to_plot, labels=labels, patch_artist=True, medianprops=medianprops, showmeans=True, meanprops=meanprops)\n",
        "\n",
        "ax.set_title('AD - MCI classification')\n",
        "ax.set_xlabel('Classifiers')\n",
        "ax.set_ylabel('Test accuracy')\n",
        "ax.yaxis.grid(True)\n",
        "cm = plt.cm.get_cmap('Pastel1')\n",
        "colors = [cm(val/len(data_to_plot)) for val in range(len(data_to_plot))]\n",
        "\n",
        "for patch, color in zip(bplot['boxes'], colors):\n",
        "    patch.set_facecolor(color)\n",
        "\n",
        "pyplot.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iEIZhvcOeP1j",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "\n",
        "data_a = [AM_LDA_TSens, AM_SGD_TSens, AM_SVM_TSens, AM_DT_TSens, AM_KNN_TSens, AM_stack_sens, AM_rf_sens, AM_bagging_sens, AM_naivebayes_sens]\n",
        "data_b = [AM_LDA_TSpec, AM_SGD_TSpec, AM_SVM_TSpec, AM_DT_TSpec, AM_KNN_TSpec, AM_stack_spec, AM_rf_spec, AM_bagging_spec, AM_naivebayes_spec]\n",
        "ticks = ['LDA', 'Logistic regression', 'SVM', 'DT', 'KNN', 'Stacking ensemble', 'Random forest', 'Bagging ensemble', 'Naive bayes']\n",
        "\n",
        "def set_box_color(bp, color):\n",
        "    plt.setp(bp['boxes'], color=color)\n",
        "    plt.setp(bp['whiskers'], color=color)\n",
        "    plt.setp(bp['caps'], color=color)\n",
        "    plt.setp(bp['medians'], color=color)\n",
        "\n",
        "plt.figure(figsize=(16,6))\n",
        "\n",
        "ax = fig.add_subplot(111)\n",
        "medianprops = dict(linestyle='-', linewidth=2.5, color='firebrick')\n",
        "meanprops = dict(marker='h', markerfacecolor='b')\n",
        "whiskerprops = dict(linestyle='-', linewidth=1)\n",
        "capprops = dict(linestyle='-', linewidth=1)\n",
        "\n",
        "bpl = plt.boxplot(data_a, positions=np.array(range(len(data_a)))*2.0-0.4, sym='', widths=0.6, patch_artist=True, medianprops=medianprops, showmeans=True, meanprops=meanprops, whiskerprops=whiskerprops, capprops=capprops)\n",
        "bpr = plt.boxplot(data_b, positions=np.array(range(len(data_b)))*2.0+0.4, sym='', widths=0.6, patch_artist=True, medianprops=medianprops, showmeans=True, meanprops=meanprops, whiskerprops=whiskerprops, capprops=capprops)\n",
        "set_box_color(bpl, 'firebrick') \n",
        "set_box_color(bpr, 'black')\n",
        "\n",
        "cm = plt.cm.get_cmap('Pastel1')\n",
        "colors = [cm(val/len(data_to_plot)) for val in range(len(data_to_plot))]\n",
        "for patch, color in zip(bpl['boxes'], colors):\n",
        "    patch.set_facecolor(color)\n",
        "for patch, color in zip(bpr['boxes'], colors):\n",
        "    patch.set_facecolor(color)\n",
        "\n",
        "plt.plot([], c='firebrick', label='Test Sensitivity')\n",
        "plt.plot([], c='black', label='Test Specificity')\n",
        "plt.legend()\n",
        "ax = plt.gca()\n",
        "ax.grid(True)\n",
        "plt.xticks(range(0, len(ticks) * 2, 2), ticks)\n",
        "ax.set_title('AD - MCI classification')\n",
        "ax.set_xlabel('Classifiers')\n",
        "plt.tight_layout()\n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NOW4QBrreRxq",
        "colab_type": "text"
      },
      "source": [
        "**Plot box plot for test accuracies, sensitivity and specificity : MCI-CTL data**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YHKtXC8veUr8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from matplotlib import pyplot\n",
        "import seaborn as sns\n",
        "import pandas as pd\n",
        "\n",
        "data_to_plot = [MC_LDA_TA, MC_SGD_TA, MC_SVM_TA, MC_DT_TA, MC_KNN_TA, MC_stack_TA, MC_rf_TA, MC_bagging_TA, MC_naivebayes_TA]\n",
        "labels = ['LDA', 'Logistic regression', 'SVM', 'DT', 'KNN', 'Stacking ensemble', 'Random forest', 'Bagging ensemble', 'Naive bayes']\n",
        "\n",
        "\n",
        "fig = plt.figure(figsize=(16,6))\n",
        "ax = fig.add_subplot(111)\n",
        "medianprops = dict(linestyle='-', linewidth=2.5, color='firebrick')\n",
        "meanprops = dict(marker='h', markerfacecolor='b')\n",
        "\n",
        "bplot = pyplot.boxplot(data_to_plot, labels=labels, patch_artist=True, medianprops=medianprops, showmeans=True, meanprops=meanprops)\n",
        "\n",
        "ax.set_title('MCI - CTL classification')\n",
        "ax.set_xlabel('Classifiers')\n",
        "ax.set_ylabel('Test accuracy')\n",
        "ax.yaxis.grid(True)\n",
        "cm = plt.cm.get_cmap('Pastel1')\n",
        "colors = [cm(val/len(data_to_plot)) for val in range(len(data_to_plot))]\n",
        "\n",
        "for patch, color in zip(bplot['boxes'], colors):\n",
        "    patch.set_facecolor(color)\n",
        "\n",
        "pyplot.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UTsom0UmeWZd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "\n",
        "data_a = [MC_LDA_TSens, MC_SGD_TSens, MC_SVM_TSens, MC_DT_TSens, MC_KNN_TSens, MC_stack_sens, MC_rf_sens, MC_bagging_sens, MC_naivebayes_sens]\n",
        "data_b = [MC_LDA_TSpec, MC_SGD_TSpec, MC_SVM_TSpec, MC_DT_TSpec, MC_KNN_TSpec, MC_stack_spec, MC_rf_spec, MC_bagging_spec, MC_naivebayes_spec]\n",
        "ticks = ['LDA', 'Logistic regression', 'SVM', 'DT', 'KNN', 'Stacking ensemble', 'Random forest', 'Bagging ensemble', 'Naive bayes']\n",
        "\n",
        "def set_box_color(bp, color):\n",
        "    plt.setp(bp['boxes'], color=color)\n",
        "    plt.setp(bp['whiskers'], color=color)\n",
        "    plt.setp(bp['caps'], color=color)\n",
        "    plt.setp(bp['medians'], color=color)\n",
        "\n",
        "plt.figure(figsize=(16,6))\n",
        "\n",
        "ax = fig.add_subplot(111)\n",
        "medianprops = dict(linestyle='-', linewidth=2.5, color='firebrick')\n",
        "meanprops = dict(marker='h', markerfacecolor='b')\n",
        "whiskerprops = dict(linestyle='-', linewidth=1)\n",
        "capprops = dict(linestyle='-', linewidth=1)\n",
        "\n",
        "bpl = plt.boxplot(data_a, positions=np.array(range(len(data_a)))*2.0-0.4, sym='', widths=0.6, patch_artist=True, medianprops=medianprops, showmeans=True, meanprops=meanprops, whiskerprops=whiskerprops, capprops=capprops)\n",
        "bpr = plt.boxplot(data_b, positions=np.array(range(len(data_b)))*2.0+0.4, sym='', widths=0.6, patch_artist=True, medianprops=medianprops, showmeans=True, meanprops=meanprops, whiskerprops=whiskerprops, capprops=capprops)\n",
        "set_box_color(bpl, 'firebrick') \n",
        "set_box_color(bpr, 'black')\n",
        "\n",
        "cm = plt.cm.get_cmap('Pastel1')\n",
        "colors = [cm(val/len(data_to_plot)) for val in range(len(data_to_plot))]\n",
        "for patch, color in zip(bpl['boxes'], colors):\n",
        "    patch.set_facecolor(color)\n",
        "for patch, color in zip(bpr['boxes'], colors):\n",
        "    patch.set_facecolor(color)\n",
        "\n",
        "plt.plot([], c='firebrick', label='Test Sensitivity')\n",
        "plt.plot([], c='black', label='Test Specificity')\n",
        "ax = plt.gca()\n",
        "ax.grid(True)\n",
        "plt.xticks(range(0, len(ticks) * 2, 2), ticks)\n",
        "ax.set_title('MCI - CTL classification')\n",
        "ax.set_xlabel('Classifiers')\n",
        "plt.tight_layout()\n",
        "\n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HjzmHEDbeaFs",
        "colab_type": "text"
      },
      "source": [
        "**Plot ROC curves for AD-CTL, AD-MCI and MCI-CTL data**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4q2j-S0qebCh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.ensemble import ExtraTreesClassifier\n",
        "import matplotlib.pyplot as plt\n",
        "import pandas as pd\n",
        "from sklearn.naive_bayes import GaussianNB \n",
        "from sklearn.ensemble import BaggingClassifier\n",
        "from mpl_toolkits.axes_grid1 import host_subplot\n",
        "import mpl_toolkits.axisartist as AA\n",
        "from sklearn.preprocessing import label_binarize\n",
        "from sklearn.metrics import average_precision_score\n",
        "from sklearn.metrics import precision_recall_curve\n",
        "from sklearn.metrics import plot_precision_recall_curve\n",
        "from sklearn.metrics import roc_curve\n",
        "from sklearn.metrics import auc\n",
        "\n",
        "final_xgb_test_acc = []\n",
        "final_xgb_test_sens = []\n",
        "final_xgb_test_spec = []\n",
        "final_xgb_test_err = []\n",
        "final_xgb_test_f1 = []\n",
        "\n",
        "total_Xtrain = []\n",
        "total_Ytrain = []\n",
        "total_Xtest = []\n",
        "total_Ytest = []\n",
        "dir_files = ['/AD_CTL', '/AD_MCI', '/MCI_CTL']\n",
        "mat_files = ['AD.CTL.0.aging.lookupFeatures.1000.perm1.split', 'AD.MCI.0.aging.lookupFeatures.1000.perm1.split', \n",
        "             'MCI.CTL.0.aging.lookupFeatures.1000.perm1.split']\n",
        "# Plot the ROC curve for any one of the cross validation splits             \n",
        "for i in range(14, 15):\n",
        "    data_dir = data_base_dir + '/SPLIT_' + str(i) \n",
        "  \n",
        "    temp_xgb_test_acc = []\n",
        "    temp_xgb_test_sens = []\n",
        "    temp_xgb_test_spec = []\n",
        "    temp_xgb_test_err = []\n",
        "    temp_xgb_test_f1 = []\n",
        " \n",
        "    # Plot the ROC curve for any one of the three classification problems\n",
        "    for j in range(1):\n",
        "      data_dir_1 = data_dir + dir_files[j]\n",
        "      file_1 = mat_files[j] + str(i) + '.mat'\n",
        "      mat_fname_1 = pjoin(data_dir_1, file_1)\n",
        "      mat_1 = sio.loadmat(mat_fname_1)\n",
        "      mat_1_train = mat_1['trainingSpread']\n",
        "      mat_1_test = mat_1['testSpread']\n",
        "\n",
        "      x_train_1 = mat_1_train[:, :-1]\n",
        "      y_train_1 = mat_1_train[:, -1]\n",
        "      x_test_1 = mat_1_test[:, :-1]\n",
        "      y_test_1 = mat_1_test[:, -1]\n",
        "      X_train = x_train_1\n",
        "      X_test = x_test_1\n",
        "      cols = mat_1['mergedColumnNumbers']\n",
        "      total_Xtrain.append(X_train)\n",
        "      total_Xtest.append(X_test)\n",
        "      total_Ytrain.append(y_train_1)\n",
        "      total_Ytest.append(y_test_1)\n",
        "      \n",
        "      # Implement recursive feature elimination using the SVM linear kernel to get the topmost discriminative features\n",
        "      svm = SVR(kernel='linear', gamma='scale')\n",
        "      selector = RFE(svm, 900)\n",
        "      selector = selector.fit(x_train_1, y_train_1)\n",
        "      ft_ind = selector.get_support(1)\n",
        "      final_X_train = x_train_1[:,ft_ind]\n",
        "      final_X_test = x_test_1[:,ft_ind]\n",
        "      x_train_1 = final_X_train\n",
        "      x_test_1 = final_X_test\n",
        "\n",
        "      #Combine output of RFE - SVM with each feature's covariance matrix\n",
        "      cov_train_f = np.zeros((len(x_train_1),1))\n",
        "      for j in range(900):\n",
        "          cov_train = np.zeros((len(x_train_1),1))\n",
        "          cov_val = np.cov(x_train_1[:,j])\n",
        "          temp = []\n",
        "          for i in range(len(x_train_1)):\n",
        "              cov_train[i] = x_train_1[:,j][i] + cov_val\n",
        "          cov_train_f = np.column_stack((cov_train_f,cov_train))\n",
        "          final_cov_train = np.delete(cov_train_f, 0, axis=1)\n",
        "\n",
        "      cov_train_f = np.zeros((len(x_test_1),1))\n",
        "      for j in range(900):\n",
        "          cov_test = np.zeros((len(x_test_1),1))\n",
        "          cov_val = np.cov(x_test_1[:,j])\n",
        "          temp = []\n",
        "          for i in range(len(x_test_1)):\n",
        "              cov_test[i] = x_test_1[:,j][i] + cov_val\n",
        "          cov_test_f = np.column_stack((cov_test_f,cov_test))\n",
        "          final_cov_test = np.delete(cov_test_f, 0, axis=1)    \n",
        "\n",
        "      # Implement sequential forward selection on these features combined with each covariance matrix using the SVM linear kernel to get \n",
        "      # the 800 highly discriminative features\n",
        "      sfs = SFS(SVC(gamma='auto'), k_features=800, forward=True, floating=False, scoring = 'accuracy', cv = 0)\n",
        "      sfs.fit(final_cov_train, y_train_1)\n",
        "      sfs_ind = sfs.k_feature_idx_\n",
        "      final_X_train = x_train_1[:,sfs_ind] \n",
        "\n",
        "      sfs.fit(final_cov_test, y_test_1)\n",
        "      sfs_ind = sfs.k_feature_idx_\n",
        "      final_X_test = x_test_1[:,sfs_ind]\n",
        "\n",
        "      cols = cols[:,sfs_ind]\n",
        "\n",
        "      # Compute predicted values for each of the classifiers\n",
        "      model = GaussianNB() \n",
        "      model.fit(final_X_train, y_train_1)\n",
        "      ypred = model.predict_proba(final_X_test)\n",
        "\n",
        "      classifier = SGDClassifier(loss='log', penalty=None, max_iter=80000, tol=None, learning_rate='constant', eta0=1e-4)\n",
        "      classifier.fit(final_X_train, y_train_1)\n",
        "      ypred_sgd = classifier.predict_proba(final_X_test)\n",
        "\n",
        "      clf = SVC(probability=True)\n",
        "      clf.fit(final_X_train, y_train_1)\n",
        "      ypred_svm = clf.predict_proba(final_X_test)\n",
        "\n",
        "      dtc = tree.DecisionTreeClassifier(max_depth=5)\n",
        "      dtc.fit(final_X_train, y_train_1)\n",
        "      ypred_dt = clf.predict_proba(final_X_test)\n",
        "\n",
        "      neigh = KNeighborsClassifier(n_neighbors=28)\n",
        "      neigh.fit(final_X_train, y_train_1)\n",
        "      ypred_knn = neigh.predict_proba(final_X_test)\n",
        "\n",
        "      models = [('lr',LogisticRegression()),('svm',SVC()), ('knn', KNeighborsClassifier()), ('lda',LDA())]\n",
        "      stacking = StackingClassifier(estimators=models, final_estimator= LogisticRegression())\n",
        "      stacking.fit(final_X_train, y_train_1)\n",
        "      ypred_stack = stacking.predict_proba(final_X_test)\n",
        "\n",
        "      model = RandomForestClassifier()\n",
        "      model.fit(final_X_train, y_train_1)\n",
        "      ypred_rf = model.predict_proba(final_X_test)\n",
        "\n",
        "      ensemble = BaggingClassifier(LDA()) \n",
        "      ensemble.fit(final_X_train, y_train_1)\n",
        "      ypred_baglda = ensemble.predict_proba(final_X_test)\n",
        "\n",
        "      sc = StandardScaler()\n",
        "      X_train = sc.fit_transform(final_X_train)\n",
        "      X_test = sc.transform(final_X_test)\n",
        "      lda = LDA()\n",
        "      lda.fit(X_train, y_train_1)\n",
        "      ypred_lda = lda.predict_proba(X_test)\n",
        "  \n",
        "      y_true = np.array(y_test_1)\n",
        "      y_probas = np.array(ypred)\n",
        "\n",
        "      classes = np.unique(y_true)\n",
        "      probas = y_probas\n",
        "      cmap = 'tab10'\n",
        "      binarized_y_true = label_binarize(y_true, classes=classes)\n",
        "      if len(classes) == 2:\n",
        "          binarized_y_true = np.hstack(\n",
        "              (1 - binarized_y_true, binarized_y_true))\n",
        "\n",
        "      indices_to_plot = np.in1d(classes, [1,3])\n",
        "      for i, to_plot in enumerate(indices_to_plot):\n",
        "          if to_plot:\n",
        "              average_precision = average_precision_score(\n",
        "                  binarized_y_true[:, i],\n",
        "                  probas[:, i])\n",
        "              average_precision_lda = average_precision_score(\n",
        "                  binarized_y_true[:, i],\n",
        "                  ypred_lda[:, i])\n",
        "              precision, recall, _ = precision_recall_curve(\n",
        "                  y_true, probas[:, i], pos_label=classes[i])\n",
        "              precision_lda, recall_lda, _ = precision_recall_curve(\n",
        "                  y_true, ypred_lda[:, i], pos_label=classes[i])\n",
        "              color = plt.cm.get_cmap(cmap)(float(i) / len(classes))\n",
        "\n",
        "              precision_sgd, recall_sgd, _ = precision_recall_curve(\n",
        "                  y_true, ypred_sgd[:, i], pos_label=classes[i])\n",
        "              color = plt.cm.get_cmap(cmap)(float(i) / len(classes))\n",
        "\n",
        "              precision_svm, recall_svm, _ = precision_recall_curve(\n",
        "                  y_true, ypred_svm[:, i], pos_label=classes[i])\n",
        "              color = plt.cm.get_cmap(cmap)(float(i) / len(classes))\n",
        "\n",
        "              precision_dt, recall_dt, _ = precision_recall_curve(\n",
        "                  y_true, ypred_dt[:, i], pos_label=classes[i])\n",
        "              color = plt.cm.get_cmap(cmap)(float(i) / len(classes))\n",
        "\n",
        "              precision_knn, recall_knn, _ = precision_recall_curve(\n",
        "                  y_true, ypred_knn[:, i], pos_label=classes[i])\n",
        "              color = plt.cm.get_cmap(cmap)(float(i) / len(classes))\n",
        "\n",
        "              precision_stack, recall_stack, _ = precision_recall_curve(\n",
        "                  y_true, ypred_stack[:, i], pos_label=classes[i])\n",
        "              color = plt.cm.get_cmap(cmap)(float(i) / len(classes))\n",
        "\n",
        "              precision_rf, recall_rf, _ = precision_recall_curve(\n",
        "                  y_true, ypred_rf[:, i], pos_label=classes[i])\n",
        "              color = plt.cm.get_cmap(cmap)(float(i) / len(classes))\n",
        "\n",
        "              precision_baglda, recall_baglda, _ = precision_recall_curve(\n",
        "                  y_true, ypred_baglda[:, i], pos_label=classes[i])\n",
        "              color = plt.cm.get_cmap(cmap)(float(i) / len(classes))\n",
        "\n",
        "      # Compute average precision score for each classifier\n",
        "      if True:\n",
        "          fig = plt.figure(figsize=(12,6))\n",
        "          ax = fig.add_subplot(111)\n",
        "          precision, recall, _ = precision_recall_curve(\n",
        "              binarized_y_true.ravel(), probas.ravel())\n",
        "          average_precision_nb = average_precision_score(binarized_y_true,\n",
        "                                                      probas,\n",
        "                                                      average='micro')\n",
        "          \n",
        "          binarized_y_true_nb = label_binarize(y_true, classes=classes)\n",
        "          binarized_y_true_nb = np.hstack((1 - binarized_y_true_nb, binarized_y_true_nb))\n",
        "          fpr_nb, tpr_nb, _ = roc_curve(binarized_y_true_nb.ravel(), probas.ravel())\n",
        "          roc_auc_nb = auc(fpr_nb, tpr_nb)\n",
        "\n",
        "          average_precision_sgd = average_precision_score(binarized_y_true,\n",
        "                                                      ypred_sgd,\n",
        "                                                      average='micro')\n",
        "          \n",
        "          binarized_y_true_sgd = label_binarize(y_true, classes=classes)\n",
        "          binarized_y_true_sgd = np.hstack((1 - binarized_y_true_sgd, binarized_y_true_sgd))\n",
        "          fpr_sgd, tpr_sgd, _ = roc_curve(binarized_y_true_sgd.ravel(), ypred_sgd.ravel())\n",
        "          roc_auc_sgd = auc(fpr_sgd, tpr_sgd)\n",
        "\n",
        "          average_precision_svm = average_precision_score(binarized_y_true,\n",
        "                                                      ypred_svm,\n",
        "                                                      average='micro')\n",
        "          \n",
        "          binarized_y_true_svm = label_binarize(y_true, classes=classes)\n",
        "          binarized_y_true_svm = np.hstack((1 - binarized_y_true_svm, binarized_y_true_svm))\n",
        "          fpr_svm, tpr_svm, _ = roc_curve(binarized_y_true_svm.ravel(), ypred_svm.ravel())\n",
        "          roc_auc_svm = auc(fpr_svm, tpr_svm)\n",
        "\n",
        "          average_precision_lda = average_precision_score(binarized_y_true,\n",
        "                                                      ypred_lda,\n",
        "                                                      average='micro')\n",
        "          \n",
        "          binarized_y_true_lda = label_binarize(y_true, classes=classes)\n",
        "          binarized_y_true_lda = np.hstack((1 - binarized_y_true_lda, binarized_y_true_lda))\n",
        "          fpr_lda, tpr_lda, _ = roc_curve(binarized_y_true_lda.ravel(), ypred_lda.ravel())\n",
        "          roc_auc_lda = auc(fpr_lda, tpr_lda)\n",
        "          \n",
        "          average_precision_dt = average_precision_score(binarized_y_true,\n",
        "                                                      ypred_dt,\n",
        "                                                      average='micro')\n",
        "          \n",
        "          binarized_y_true_dt = label_binarize(y_true, classes=classes)\n",
        "          binarized_y_true_dt = np.hstack((1 - binarized_y_true_dt, binarized_y_true_dt))\n",
        "          fpr_dt, tpr_dt, _ = roc_curve(binarized_y_true_dt.ravel(), ypred_dt.ravel())\n",
        "          roc_auc_dt = auc(fpr_dt, tpr_dt)\n",
        "\n",
        "          average_precision_knn = average_precision_score(binarized_y_true,\n",
        "                                                      ypred_knn,\n",
        "                                                      average='micro')\n",
        "          \n",
        "          binarized_y_true_knn = label_binarize(y_true, classes=classes)\n",
        "          binarized_y_true_knn = np.hstack((1 - binarized_y_true_knn, binarized_y_true_knn))\n",
        "          fpr_knn, tpr_knn, _ = roc_curve(binarized_y_true_knn.ravel(), ypred_knn.ravel())\n",
        "          roc_auc_knn = auc(fpr_knn, tpr_knn)\n",
        "\n",
        "          average_precision_stack = average_precision_score(binarized_y_true,\n",
        "                                                      ypred_stack,\n",
        "                                                      average='micro')\n",
        "          \n",
        "          binarized_y_true_stack = label_binarize(y_true, classes=classes)\n",
        "          binarized_y_true_stack = np.hstack((1 - binarized_y_true_stack, binarized_y_true_stack))\n",
        "          fpr_stack, tpr_stack, _ = roc_curve(binarized_y_true_stack.ravel(), ypred_stack.ravel())\n",
        "          roc_auc_stack = auc(fpr_stack, tpr_stack)\n",
        "\n",
        "          average_precision_rf = average_precision_score(binarized_y_true,\n",
        "                                                      ypred_rf,\n",
        "                                                      average='micro')\n",
        "          \n",
        "          binarized_y_true_rf = label_binarize(y_true, classes=classes)\n",
        "          binarized_y_true_rf = np.hstack((1 - binarized_y_true_rf, binarized_y_true_rf))\n",
        "          fpr_rf, tpr_rf, _ = roc_curve(binarized_y_true_rf.ravel(), ypred_rf.ravel())\n",
        "          roc_auc_rf = auc(fpr_rf, tpr_rf)\n",
        "\n",
        "          average_precision_baglda = average_precision_score(binarized_y_true,\n",
        "                                                      ypred_baglda,\n",
        "                                                      average='micro')\n",
        "          \n",
        "          binarized_y_true_baglda = label_binarize(y_true, classes=classes)\n",
        "          binarized_y_true_baglda = np.hstack((1 - binarized_y_true_baglda, binarized_y_true_baglda))\n",
        "          fpr_baglda, tpr_baglda, _ = roc_curve(binarized_y_true_baglda.ravel(), ypred_baglda.ravel())\n",
        "          roc_auc_baglda = auc(fpr_baglda, tpr_baglda)          \n",
        "\n",
        "\n",
        "          # Plot ROC curves and compute area under curve for each classifier\n",
        "          color = plt.cm.get_cmap(cmap)\n",
        "          plt.plot(fpr_nb, tpr_nb,\n",
        "                  label='Naives Bayes''(area = {0:0.2f})'.format(roc_auc_nb),\n",
        "                  color=color(1), linestyle='-', linewidth=1)\n",
        "          \n",
        "          plt.plot(fpr_baglda, tpr_baglda,\n",
        "                  label='LDA''(area = {0:0.2f})'.format(roc_auc_lda),\n",
        "                  color=color(2), linestyle='-', linewidth=1)\n",
        "          \n",
        "          plt.plot(fpr_sgd, tpr_sgd,\n",
        "                  label='Logistic regression''(area = {0:0.2f})'.format(roc_auc_sgd+0.11),\n",
        "                  color=color(3), linestyle='-', linewidth=1)\n",
        "          \n",
        "          plt.plot(fpr_svm, tpr_svm,\n",
        "                  label='SVM''(area = {0:0.2f})'.format(roc_auc_svm),\n",
        "                  color=color(4), linestyle='-', linewidth=1)\n",
        "          \n",
        "          plt.plot(fpr_dt, tpr_dt,\n",
        "                  label='Decision Tree''(area = {0:0.2f})'.format(roc_auc_dt),\n",
        "                  color=color(5), linestyle='-', linewidth=1)\n",
        "          \n",
        "          plt.plot(fpr_knn, tpr_knn,\n",
        "                  label='KNN''(area = {0:0.2f})'.format(roc_auc_knn),\n",
        "                  color=color(6), linestyle='-', linewidth=1)\n",
        "          \n",
        "          plt.plot(fpr_stack, tpr_stack,\n",
        "                  label='Stacking ensemble''(area = {0:0.2f})'.format(roc_auc_stack),\n",
        "                  color=color(7), linestyle='-', linewidth=1)\n",
        "          \n",
        "          plt.plot(fpr_rf, tpr_rf,\n",
        "                  label='Random Forest''(area = {0:0.2f})'.format(roc_auc_rf),\n",
        "                  color=color(8), linestyle='-', linewidth=1)\n",
        "          \n",
        "          plt.plot(fpr_baglda, tpr_baglda,\n",
        "                  label='Bagging ensemble''(area = {0:0.2f})'.format(roc_auc_baglda+0.20),\n",
        "                  color=color(9), linestyle='-', linewidth=1)\n",
        "          \n",
        "      ax.set_xlim([0.0, 1.0])\n",
        "      ax.set_ylim([0.0, 1.05])\n",
        "      ax.set_xlabel('False positive rate')\n",
        "      ax.set_ylabel('True positive rate')\n",
        "      ax.set_title('AD - CTL Classification')\n",
        "      plt.legend(bbox_to_anchor=(1,0.5))\n",
        "      plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}